diff --git a/.nojekyll b/.nojekyll
index 6f13e3e1c..192c28161 100644
--- a/.nojekyll
+++ b/.nojekyll
@@ -1 +1 @@
-ed1c890d
\ No newline at end of file
+647e5ce9
\ No newline at end of file
diff --git a/admins/cluster-config.html b/admins/cluster-config.html
index 934dfd367..5daf1baca 100644
--- a/admins/cluster-config.html
+++ b/admins/cluster-config.html
@@ -188,12 +188,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/credentials.html b/admins/credentials.html
index 3b1465fc1..2032b7efc 100644
--- a/admins/credentials.html
+++ b/admins/credentials.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/calendar-scaler.html b/admins/howto/calendar-scaler.html
index 6c51bde51..eabb530ce 100644
--- a/admins/howto/calendar-scaler.html
+++ b/admins/howto/calendar-scaler.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/cheatsheet.html b/admins/howto/cheatsheet.html
index d8b53142d..8237f79c0 100644
--- a/admins/howto/cheatsheet.html
+++ b/admins/howto/cheatsheet.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/clusterswitch.html b/admins/howto/clusterswitch.html
index 7c3dfcee8..11edee218 100644
--- a/admins/howto/clusterswitch.html
+++ b/admins/howto/clusterswitch.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/core-pool.html b/admins/howto/core-pool.html
index a3a76d7e6..dc5ed311e 100644
--- a/admins/howto/core-pool.html
+++ b/admins/howto/core-pool.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/course-config.html b/admins/howto/course-config.html
index 3d1d11f8b..3592ed30d 100644
--- a/admins/howto/course-config.html
+++ b/admins/howto/course-config.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/delete-hub.html b/admins/howto/delete-hub.html
index ed6ede9e7..cd3412233 100644
--- a/admins/howto/delete-hub.html
+++ b/admins/howto/delete-hub.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/dns.html b/admins/howto/dns.html
index 6d637a879..08557f6df 100644
--- a/admins/howto/dns.html
+++ b/admins/howto/dns.html
@@ -154,12 +154,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/documentation.html b/admins/howto/documentation.html
index d19c8bb16..b14ffa912 100644
--- a/admins/howto/documentation.html
+++ b/admins/howto/documentation.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/github-token.html b/admins/howto/github-token.html
index e3b8d219c..aa17ebd8b 100644
--- a/admins/howto/github-token.html
+++ b/admins/howto/github-token.html
@@ -154,12 +154,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/google-sheets.html b/admins/howto/google-sheets.html
index 51507e3d7..732cf0337 100644
--- a/admins/howto/google-sheets.html
+++ b/admins/howto/google-sheets.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/index.html b/admins/howto/index.html
index 26864f2ac..d3d89bb06 100644
--- a/admins/howto/index.html
+++ b/admins/howto/index.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/managing-multiple-user-image-repos.html b/admins/howto/managing-multiple-user-image-repos.html
index 9b56a4f11..7230178f8 100644
--- a/admins/howto/managing-multiple-user-image-repos.html
+++ b/admins/howto/managing-multiple-user-image-repos.html
@@ -154,12 +154,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/new-hub.html b/admins/howto/new-hub.html
index caff60dd7..24fc0cda4 100644
--- a/admins/howto/new-hub.html
+++ b/admins/howto/new-hub.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/new-image.html b/admins/howto/new-image.html
index 2b536f028..ccd2ce573 100644
--- a/admins/howto/new-image.html
+++ b/admins/howto/new-image.html
@@ -154,12 +154,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/new-packages.html b/admins/howto/new-packages.html
index cd6db8140..fca16daea 100644
--- a/admins/howto/new-packages.html
+++ b/admins/howto/new-packages.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/prometheus-grafana.html b/admins/howto/prometheus-grafana.html
index a57493b19..552f2148d 100644
--- a/admins/howto/prometheus-grafana.html
+++ b/admins/howto/prometheus-grafana.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/rebuild-hub-image.html b/admins/howto/rebuild-hub-image.html
index b39aa27a1..062e65ed5 100644
--- a/admins/howto/rebuild-hub-image.html
+++ b/admins/howto/rebuild-hub-image.html
@@ -154,12 +154,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/rebuild-postgres-image.html b/admins/howto/rebuild-postgres-image.html
index 004680260..745dee370 100644
--- a/admins/howto/rebuild-postgres-image.html
+++ b/admins/howto/rebuild-postgres-image.html
@@ -154,12 +154,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/remove-users-orm.html b/admins/howto/remove-users-orm.html
index 0ce2dd390..5eb128b3f 100644
--- a/admins/howto/remove-users-orm.html
+++ b/admins/howto/remove-users-orm.html
@@ -154,12 +154,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/howto/transition-image.html b/admins/howto/transition-image.html
index 8347f1ffc..79074bd27 100644
--- a/admins/howto/transition-image.html
+++ b/admins/howto/transition-image.html
@@ -188,12 +188,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/index.html b/admins/index.html
index bfec184d8..da297f681 100644
--- a/admins/index.html
+++ b/admins/index.html
@@ -152,12 +152,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/pre-reqs.html b/admins/pre-reqs.html
index c2ccf3889..2488dddc2 100644
--- a/admins/pre-reqs.html
+++ b/admins/pre-reqs.html
@@ -31,7 +31,7 @@
 <script src="../site_libs/quarto-search/quarto-search.js"></script>
 <meta name="quarto:offset" content="../">
 <link href="../admins/structure.html" rel="next">
-<link href="../users/authentication.html" rel="prev">
+<link href="../users/hubs.html" rel="prev">
 <script src="../site_libs/quarto-html/quarto.js"></script>
 <script src="../site_libs/quarto-html/popper.min.js"></script>
 <script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
@@ -863,8 +857,8 @@ <h2 class="anchored" data-anchor-id="full">Full</h2>
 </script>
 <nav class="page-navigation">
   <div class="nav-page nav-page-previous">
-      <a href="../users/authentication.html" class="pagination-link" aria-label="User Authentication">
-        <i class="bi bi-arrow-left-short"></i> <span class="nav-page-text">User Authentication</span>
+      <a href="../users/hubs.html" class="pagination-link" aria-label="JupyterHub Deployments">
+        <i class="bi bi-arrow-left-short"></i> <span class="nav-page-text">JupyterHub Deployments</span>
       </a>          
   </div>
   <div class="nav-page nav-page-next">
diff --git a/admins/storage.html b/admins/storage.html
index b97ebb00b..422095464 100644
--- a/admins/storage.html
+++ b/admins/storage.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/admins/structure.html b/admins/structure.html
index 685b5c133..18bdc5335 100644
--- a/admins/structure.html
+++ b/admins/structure.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html b/incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html
index a1131b09e..c0d3238dc 100644
--- a/incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html
+++ b/incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html
@@ -189,12 +189,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-02-09-datahub-db-outage.html b/incidents/2017-02-09-datahub-db-outage.html
index 9fd8a3ea3..4ebf5b53d 100644
--- a/incidents/2017-02-09-datahub-db-outage.html
+++ b/incidents/2017-02-09-datahub-db-outage.html
@@ -187,12 +187,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-02-24-autoscaler-incident.html b/incidents/2017-02-24-autoscaler-incident.html
index 6eefa67dc..8a15e4b92 100644
--- a/incidents/2017-02-24-autoscaler-incident.html
+++ b/incidents/2017-02-24-autoscaler-incident.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-02-24-proxy-death-incident.html b/incidents/2017-02-24-proxy-death-incident.html
index f82977ba9..486338ea8 100644
--- a/incidents/2017-02-24-proxy-death-incident.html
+++ b/incidents/2017-02-24-proxy-death-incident.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-03-06-helm-config-image-mismatch.html b/incidents/2017-03-06-helm-config-image-mismatch.html
index 2d74095c7..786682163 100644
--- a/incidents/2017-03-06-helm-config-image-mismatch.html
+++ b/incidents/2017-03-06-helm-config-image-mismatch.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-03-20-too-many-volumes.html b/incidents/2017-03-20-too-many-volumes.html
index 4737b0654..eedbde7f9 100644
--- a/incidents/2017-03-20-too-many-volumes.html
+++ b/incidents/2017-03-20-too-many-volumes.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-03-23-kernel-deaths-incident.html b/incidents/2017-03-23-kernel-deaths-incident.html
index ef3056e2d..f6e5c7485 100644
--- a/incidents/2017-03-23-kernel-deaths-incident.html
+++ b/incidents/2017-03-23-kernel-deaths-incident.html
@@ -187,12 +187,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-04-03-cluster-full-incident.html b/incidents/2017-04-03-cluster-full-incident.html
index 54bd8ae1f..53d59d849 100644
--- a/incidents/2017-04-03-cluster-full-incident.html
+++ b/incidents/2017-04-03-cluster-full-incident.html
@@ -187,12 +187,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-05-09-gce-billing.html b/incidents/2017-05-09-gce-billing.html
index 09753cf4b..01c7cff52 100644
--- a/incidents/2017-05-09-gce-billing.html
+++ b/incidents/2017-05-09-gce-billing.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-10-10-hung-nodes.html b/incidents/2017-10-10-hung-nodes.html
index 75911b29b..01436cbf1 100644
--- a/incidents/2017-10-10-hung-nodes.html
+++ b/incidents/2017-10-10-hung-nodes.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2017-10-19-course-subscription-canceled.html b/incidents/2017-10-19-course-subscription-canceled.html
index 9f75635be..ae5c47d63 100644
--- a/incidents/2017-10-19-course-subscription-canceled.html
+++ b/incidents/2017-10-19-course-subscription-canceled.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2018-01-25-helm-chart-upgrade.html b/incidents/2018-01-25-helm-chart-upgrade.html
index 675e2cc6f..fe3db286c 100644
--- a/incidents/2018-01-25-helm-chart-upgrade.html
+++ b/incidents/2018-01-25-helm-chart-upgrade.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2018-01-26-hub-slow-startup.html b/incidents/2018-01-26-hub-slow-startup.html
index d986a1299..875666aa8 100644
--- a/incidents/2018-01-26-hub-slow-startup.html
+++ b/incidents/2018-01-26-hub-slow-startup.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2018-02-06-hub-db-dir.html b/incidents/2018-02-06-hub-db-dir.html
index 58ce7c134..c2893c619 100644
--- a/incidents/2018-02-06-hub-db-dir.html
+++ b/incidents/2018-02-06-hub-db-dir.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2018-02-28-hung-node.html b/incidents/2018-02-28-hung-node.html
index 3b8b83029..16a874ed4 100644
--- a/incidents/2018-02-28-hung-node.html
+++ b/incidents/2018-02-28-hung-node.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2018-06-11-course-subscription-canceled.html b/incidents/2018-06-11-course-subscription-canceled.html
index cdb8ac8c2..17d1b5257 100644
--- a/incidents/2018-06-11-course-subscription-canceled.html
+++ b/incidents/2018-06-11-course-subscription-canceled.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2019-02-25-k8s-api-server-down.html b/incidents/2019-02-25-k8s-api-server-down.html
index e56357a7b..885ee5ab7 100644
--- a/incidents/2019-02-25-k8s-api-server-down.html
+++ b/incidents/2019-02-25-k8s-api-server-down.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2019-05-01-service-account-leak.html b/incidents/2019-05-01-service-account-leak.html
index 97bfd784a..0b6e5762a 100644
--- a/incidents/2019-05-01-service-account-leak.html
+++ b/incidents/2019-05-01-service-account-leak.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2022-01-20-package-dependency-upgrade-incident.html b/incidents/2022-01-20-package-dependency-upgrade-incident.html
index f014364a6..eb88fca47 100644
--- a/incidents/2022-01-20-package-dependency-upgrade-incident.html
+++ b/incidents/2022-01-20-package-dependency-upgrade-incident.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/2024-core-node-incidents.html b/incidents/2024-core-node-incidents.html
index 684e8c5da..303385faa 100644
--- a/incidents/2024-core-node-incidents.html
+++ b/incidents/2024-core-node-incidents.html
@@ -153,12 +153,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/incidents/index.html b/incidents/index.html
index 6188902d8..98a082235 100644
--- a/incidents/index.html
+++ b/incidents/index.html
@@ -213,12 +213,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
@@ -498,7 +492,7 @@ <h1 class="title">Incident Reports</h1>
 </tr>
 </thead>
 <tbody class="list">
-<tr data-index="0" data-listing-date-sort="1486598400000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="7" data-listing-word-count-sort="1292" data-listing-title-sort="JupyterHub db manual overwrite" data-listing-filename-sort="2017-02-09-datahub-db-outage.qmd">
+<tr data-index="0" data-listing-date-sort="1486598400000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="7" data-listing-word-count-sort="1292" data-listing-title-sort="JupyterHub db manual overwrite" data-listing-filename-sort="2017-02-09-datahub-db-outage.qmd">
 <td>
 <span class="listing-date">Feb 9, 2017</span>
 </td>
@@ -506,7 +500,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-02-09-datahub-db-outage.html" class="title listing-title">JupyterHub db manual overwrite</a>
 </td>
 </tr>
-<tr data-index="1" data-listing-date-sort="1487894400000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="353" data-listing-title-sort="Custom Autoscaler gonee haywire" data-listing-filename-sort="2017-02-24-autoscaler-incident.qmd">
+<tr data-index="1" data-listing-date-sort="1487894400000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="353" data-listing-title-sort="Custom Autoscaler gonee haywire" data-listing-filename-sort="2017-02-24-autoscaler-incident.qmd">
 <td>
 <span class="listing-date">Feb 24, 2017</span>
 </td>
@@ -514,7 +508,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-02-24-autoscaler-incident.html" class="title listing-title">Custom Autoscaler gonee haywire</a>
 </td>
 </tr>
-<tr data-index="2" data-listing-date-sort="1487894400000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="628" data-listing-title-sort="Proxy eviction strands user" data-listing-filename-sort="2017-02-24-proxy-death-incident.qmd">
+<tr data-index="2" data-listing-date-sort="1487894400000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="628" data-listing-title-sort="Proxy eviction strands user" data-listing-filename-sort="2017-02-24-proxy-death-incident.qmd">
 <td>
 <span class="listing-date">Feb 24, 2017</span>
 </td>
@@ -522,7 +516,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-02-24-proxy-death-incident.html" class="title listing-title">Proxy eviction strands user</a>
 </td>
 </tr>
-<tr data-index="3" data-listing-date-sort="1488758400000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="576" data-listing-title-sort="Non-matching hub image tags cause downtime" data-listing-filename-sort="2017-03-06-helm-config-image-mismatch.qmd">
+<tr data-index="3" data-listing-date-sort="1488758400000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="576" data-listing-title-sort="Non-matching hub image tags cause downtime" data-listing-filename-sort="2017-03-06-helm-config-image-mismatch.qmd">
 <td>
 <span class="listing-date">Mar 6, 2017</span>
 </td>
@@ -530,7 +524,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-03-06-helm-config-image-mismatch.html" class="title listing-title">Non-matching hub image tags cause downtime</a>
 </td>
 </tr>
-<tr data-index="4" data-listing-date-sort="1489968000000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="580" data-listing-title-sort="Too many volumes per disk leave students stuck" data-listing-filename-sort="2017-03-20-too-many-volumes.qmd">
+<tr data-index="4" data-listing-date-sort="1489968000000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="580" data-listing-title-sort="Too many volumes per disk leave students stuck" data-listing-filename-sort="2017-03-20-too-many-volumes.qmd">
 <td>
 <span class="listing-date">Mar 20, 2017</span>
 </td>
@@ -538,7 +532,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-03-20-too-many-volumes.html" class="title listing-title">Too many volumes per disk leave students stuck</a>
 </td>
 </tr>
-<tr data-index="5" data-listing-date-sort="1490227200000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="526" data-listing-title-sort="Weird upstream ipython bug kills kernels" data-listing-filename-sort="2017-03-23-kernel-deaths-incident.qmd">
+<tr data-index="5" data-listing-date-sort="1490227200000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="526" data-listing-title-sort="Weird upstream ipython bug kills kernels" data-listing-filename-sort="2017-03-23-kernel-deaths-incident.qmd">
 <td>
 <span class="listing-date">Mar 23, 2017</span>
 </td>
@@ -546,7 +540,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-03-23-kernel-deaths-incident.html" class="title listing-title">Weird upstream ipython bug kills kernels</a>
 </td>
 </tr>
-<tr data-index="6" data-listing-date-sort="1491177600000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="290" data-listing-title-sort="Custom autoscaler does not scale up when it should" data-listing-filename-sort="2017-04-03-cluster-full-incident.qmd">
+<tr data-index="6" data-listing-date-sort="1491177600000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="290" data-listing-title-sort="Custom autoscaler does not scale up when it should" data-listing-filename-sort="2017-04-03-cluster-full-incident.qmd">
 <td>
 <span class="listing-date">Apr 3, 2017</span>
 </td>
@@ -554,7 +548,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-04-03-cluster-full-incident.html" class="title listing-title">Custom autoscaler does not scale up when it should</a>
 </td>
 </tr>
-<tr data-index="7" data-listing-date-sort="1494288000000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="333" data-listing-title-sort="Oops we forgot to pay the bill" data-listing-filename-sort="2017-05-09-gce-billing.qmd">
+<tr data-index="7" data-listing-date-sort="1494288000000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="333" data-listing-title-sort="Oops we forgot to pay the bill" data-listing-filename-sort="2017-05-09-gce-billing.qmd">
 <td>
 <span class="listing-date">May 9, 2017</span>
 </td>
@@ -562,7 +556,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-05-09-gce-billing.html" class="title listing-title">Oops we forgot to pay the bill</a>
 </td>
 </tr>
-<tr data-index="8" data-listing-date-sort="1507593600000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="5" data-listing-word-count-sort="813" data-listing-title-sort="Docker dies on a few Azure nodes" data-listing-filename-sort="2017-10-10-hung-nodes.qmd">
+<tr data-index="8" data-listing-date-sort="1507593600000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="5" data-listing-word-count-sort="813" data-listing-title-sort="Docker dies on a few Azure nodes" data-listing-filename-sort="2017-10-10-hung-nodes.qmd">
 <td>
 <span class="listing-date">Oct 10, 2017</span>
 </td>
@@ -570,7 +564,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-10-10-hung-nodes.html" class="title listing-title">Docker dies on a few Azure nodes</a>
 </td>
 </tr>
-<tr data-index="9" data-listing-date-sort="1508371200000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="746" data-listing-title-sort="Billing confusion with Azure portal causes summer hub to be lost" data-listing-filename-sort="2017-10-19-course-subscription-canceled.qmd">
+<tr data-index="9" data-listing-date-sort="1508371200000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="746" data-listing-title-sort="Billing confusion with Azure portal causes summer hub to be lost" data-listing-filename-sort="2017-10-19-course-subscription-canceled.qmd">
 <td>
 <span class="listing-date">Oct 19, 2017</span>
 </td>
@@ -578,7 +572,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2017-10-19-course-subscription-canceled.html" class="title listing-title">Billing confusion with Azure portal causes summer hub to be lost</a>
 </td>
 </tr>
-<tr data-index="10" data-listing-date-sort="1516838400000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="342" data-listing-title-sort="Accidental merge to prod brings things down" data-listing-filename-sort="2018-01-25-helm-chart-upgrade.qmd">
+<tr data-index="10" data-listing-date-sort="1516838400000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="342" data-listing-title-sort="Accidental merge to prod brings things down" data-listing-filename-sort="2018-01-25-helm-chart-upgrade.qmd">
 <td>
 <span class="listing-date">Jan 25, 2018</span>
 </td>
@@ -586,7 +580,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2018-01-25-helm-chart-upgrade.html" class="title listing-title">Accidental merge to prod brings things down</a>
 </td>
 </tr>
-<tr data-index="11" data-listing-date-sort="1516924800000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="377" data-listing-title-sort="Hub starts up very slow, causing outage for users" data-listing-filename-sort="2018-01-26-hub-slow-startup.qmd">
+<tr data-index="11" data-listing-date-sort="1516924800000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="377" data-listing-title-sort="Hub starts up very slow, causing outage for users" data-listing-filename-sort="2018-01-26-hub-slow-startup.qmd">
 <td>
 <span class="listing-date">Jan 26, 2018</span>
 </td>
@@ -594,7 +588,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2018-01-26-hub-slow-startup.html" class="title listing-title">Hub starts up very slow, causing outage for users</a>
 </td>
 </tr>
-<tr data-index="12" data-listing-date-sort="1517875200000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="284" data-listing-title-sort="Azure PD refuses to detach, causing downtime for data100" data-listing-filename-sort="2018-02-06-hub-db-dir.qmd">
+<tr data-index="12" data-listing-date-sort="1517875200000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="284" data-listing-title-sort="Azure PD refuses to detach, causing downtime for data100" data-listing-filename-sort="2018-02-06-hub-db-dir.qmd">
 <td>
 <span class="listing-date">Feb 6, 2018</span>
 </td>
@@ -602,7 +596,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2018-02-06-hub-db-dir.html" class="title listing-title">Azure PD refuses to detach, causing downtime for data100</a>
 </td>
 </tr>
-<tr data-index="13" data-listing-date-sort="1519776000000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="222" data-listing-title-sort="A node hangs, causing a subset of users to report issues" data-listing-filename-sort="2018-02-28-hung-node.qmd">
+<tr data-index="13" data-listing-date-sort="1519776000000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="222" data-listing-title-sort="A node hangs, causing a subset of users to report issues" data-listing-filename-sort="2018-02-28-hung-node.qmd">
 <td>
 <span class="listing-date">Feb 28, 2018</span>
 </td>
@@ -610,7 +604,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2018-02-28-hung-node.html" class="title listing-title">A node hangs, causing a subset of users to report issues</a>
 </td>
 </tr>
-<tr data-index="14" data-listing-date-sort="1528675200000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="338" data-listing-title-sort="Azure billing issue causes downtime" data-listing-filename-sort="2018-06-11-course-subscription-canceled.qmd">
+<tr data-index="14" data-listing-date-sort="1528675200000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="2" data-listing-word-count-sort="338" data-listing-title-sort="Azure billing issue causes downtime" data-listing-filename-sort="2018-06-11-course-subscription-canceled.qmd">
 <td>
 <span class="listing-date">Jun 11, 2018</span>
 </td>
@@ -618,7 +612,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2018-06-11-course-subscription-canceled.html" class="title listing-title">Azure billing issue causes downtime</a>
 </td>
 </tr>
-<tr data-index="15" data-listing-date-sort="1551052800000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="447" data-listing-title-sort="Azure Kubernetes API Server outage causes downtime" data-listing-filename-sort="2019-02-25-k8s-api-server-down.qmd">
+<tr data-index="15" data-listing-date-sort="1551052800000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="3" data-listing-word-count-sort="447" data-listing-title-sort="Azure Kubernetes API Server outage causes downtime" data-listing-filename-sort="2019-02-25-k8s-api-server-down.qmd">
 <td>
 <span class="listing-date">Feb 25, 2019</span>
 </td>
@@ -626,7 +620,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2019-02-25-k8s-api-server-down.html" class="title listing-title">Azure Kubernetes API Server outage causes downtime</a>
 </td>
 </tr>
-<tr data-index="16" data-listing-date-sort="1556668800000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="1" data-listing-word-count-sort="174" data-listing-title-sort="Service Account key leak incident" data-listing-filename-sort="2019-05-01-service-account-leak.qmd">
+<tr data-index="16" data-listing-date-sort="1556668800000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="1" data-listing-word-count-sort="174" data-listing-title-sort="Service Account key leak incident" data-listing-filename-sort="2019-05-01-service-account-leak.qmd">
 <td>
 <span class="listing-date">May 1, 2019</span>
 </td>
@@ -634,7 +628,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2019-05-01-service-account-leak.html" class="title listing-title">Service Account key leak incident</a>
 </td>
 </tr>
-<tr data-index="17" data-listing-date-sort="1642636800000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="767" data-listing-title-sort="Hubs throwing 505 errors" data-listing-filename-sort="2022-01-20-package-dependency-upgrade-incident.qmd">
+<tr data-index="17" data-listing-date-sort="1642636800000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="767" data-listing-title-sort="Hubs throwing 505 errors" data-listing-filename-sort="2022-01-20-package-dependency-upgrade-incident.qmd">
 <td>
 <span class="listing-date">Jan 20, 2022</span>
 </td>
@@ -642,7 +636,7 @@ <h1 class="title">Incident Reports</h1>
 <a href="../incidents/2022-01-20-package-dependency-upgrade-incident.html" class="title listing-title">Hubs throwing 505 errors</a>
 </td>
 </tr>
-<tr data-index="18" data-listing-date-sort="1706745600000" data-listing-file-modified-sort="1729746659359" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="5" data-listing-word-count-sort="874" data-listing-title-sort="Core nodes being autoscaled, configurable HTTP proxy crashes" data-listing-filename-sort="2024-core-node-incidents.qmd">
+<tr data-index="18" data-listing-date-sort="1706745600000" data-listing-file-modified-sort="1729748266831" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="5" data-listing-word-count-sort="874" data-listing-title-sort="Core nodes being autoscaled, configurable HTTP proxy crashes" data-listing-filename-sort="2024-core-node-incidents.qmd">
 <td>
 <span class="listing-date">Feb 1, 2024</span>
 </td>
diff --git a/index.html b/index.html
index eeab40a60..2ceec133e 100644
--- a/index.html
+++ b/index.html
@@ -153,12 +153,6 @@
   <a href="./users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="./users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/policy/create_policy.html b/policy/create_policy.html
index 47a5dc0bb..742d17fe5 100644
--- a/policy/create_policy.html
+++ b/policy/create_policy.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/policy/index.html b/policy/index.html
index edd2baace..546d0438d 100644
--- a/policy/index.html
+++ b/policy/index.html
@@ -152,12 +152,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/policy/policy_create_hubs.html b/policy/policy_create_hubs.html
index 20acf5882..29b1cf39e 100644
--- a/policy/policy_create_hubs.html
+++ b/policy/policy_create_hubs.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/policy/policy_deploy_mainhubs.html b/policy/policy_deploy_mainhubs.html
index c571c9e5e..d65da0ebf 100644
--- a/policy/policy_deploy_mainhubs.html
+++ b/policy/policy_deploy_mainhubs.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/policy/principles.html b/policy/principles.html
index 9cf9a1cef..2b2deeeba 100644
--- a/policy/principles.html
+++ b/policy/principles.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/policy/storage-retention.html b/policy/storage-retention.html
index d2968ec62..f40ba13c9 100644
--- a/policy/storage-retention.html
+++ b/policy/storage-retention.html
@@ -152,12 +152,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/search.json b/search.json
index 993951c01..3085c9c9f 100644
--- a/search.json
+++ b/search.json
@@ -784,351 +784,340 @@
     ]
   },
   {
-    "objectID": "incidents/2017-02-24-proxy-death-incident.html",
-    "href": "incidents/2017-02-24-proxy-death-incident.html",
-    "title": "Proxy eviction strands user",
+    "objectID": "incidents/2017-04-03-cluster-full-incident.html",
+    "href": "incidents/2017-04-03-cluster-full-incident.html",
+    "title": "Custom autoscaler does not scale up when it should",
     "section": "",
-    "text": "On the evening of Feb 23, several students started experiencing 500 errors in trying to access datahub. The proxy had died because of a known issue, and it took a while for the hub to re-add all the user routes to the proxy. Some students’ needed their servers to be manually restarted, due to a JupyterHub spawner bug that is showing up at scale. Everything was fixed in about 40 minutes."
+    "text": "On April 3, 2017, as students were returning from spring break, the cluster wasn’t big enough in time and several students had errors spawning. This was because the simple-autoscaler was ‘stuck’ on a populate call. More capacity was manually added, the pending pods were deleted & this seemed to fix the outage."
   },
   {
-    "objectID": "incidents/2017-02-24-proxy-death-incident.html#summary",
-    "href": "incidents/2017-02-24-proxy-death-incident.html#summary",
-    "title": "Proxy eviction strands user",
+    "objectID": "incidents/2017-04-03-cluster-full-incident.html#summary",
+    "href": "incidents/2017-04-03-cluster-full-incident.html#summary",
+    "title": "Custom autoscaler does not scale up when it should",
     "section": "",
-    "text": "On the evening of Feb 23, several students started experiencing 500 errors in trying to access datahub. The proxy had died because of a known issue, and it took a while for the hub to re-add all the user routes to the proxy. Some students’ needed their servers to be manually restarted, due to a JupyterHub spawner bug that is showing up at scale. Everything was fixed in about 40 minutes."
+    "text": "On April 3, 2017, as students were returning from spring break, the cluster wasn’t big enough in time and several students had errors spawning. This was because the simple-autoscaler was ‘stuck’ on a populate call. More capacity was manually added, the pending pods were deleted & this seemed to fix the outage."
   },
   {
-    "objectID": "incidents/2017-02-24-proxy-death-incident.html#timeline",
-    "href": "incidents/2017-02-24-proxy-death-incident.html#timeline",
-    "title": "Proxy eviction strands user",
+    "objectID": "incidents/2017-04-03-cluster-full-incident.html#timeline",
+    "href": "incidents/2017-04-03-cluster-full-incident.html#timeline",
+    "title": "Custom autoscaler does not scale up when it should",
     "section": "Timeline",
-    "text": "Timeline\nAll times in PST\n\n21:10:57\nThe proxy pod is evicted, due to a known issue that is currently being worked on. Users start running into issue now, with connection failures.\n\n\n21:11:04\nNew proxy pod is started by kubernetes, and starts accepting connections. However, the JupyterHub model currently has the proxy starting with no state about user routes, and so the users’ requests aren’t being routed to their notebook pods. This manifests as errors for users.\nThe hub process is supposed to poll the proxy every 300s, and repopulate the route table when it notices it is empty. The hub does this at some point in the next 300s (we do not know when), and starts repopulating the route table. As routes get added for currently users, their notebook starts working again.\n\n\n21:11:52\nThe repopulate process starts running into issues - it is making far too many http requests (to the kubernetes and proxy APIs) that it starts running into client side limits on tornado http client (which is what we use to make these requests). This causes them to time out on the request queue. We were running into https://github.com/tornadoweb/tornado/issues/1400. Not all requests fail - for those that succeed, the students are able to access their notebooks.\nThe repopulate process takes a while to process, and errors for a lot of students who are left with notebook in inconsistent state - JupyterHub thinks their notebook is running but it isn’t, or vice versa. Lots of 500s for users.\n\n\n21:14\nReports of errors start reaching the Slack channel + Piazza.\nThe repopulate process keeps being retried, and notebooks for users slowly come back. Some users are ‘stuck’ in a bad state, however - their notebook isn’t running, but JupyterHub thinks it is (or vice versa).\n\n\n21:34\nMost users are fine by now. For those still with problems, a forced delete from the admin interface + a start works, since this forces JupyterHub to really check if they’re there or not.\n\n\n22:03\nLast reported user with 500 error is fixed, and datahub is fully operational again."
+    "text": "Timeline\n\nOver spring break week\nThe cluster is scaled down to a much smaller size (7 machines), and the simple scaler is left running.\n\n\n2017-04-03 11:32\nStudents report datahub isn’t working on Piazza, and lots of Pods in PENDING state.\nDoing a kubectl --namespace=datahub describe pod &lt;pod-name&gt; said the pod was unschedulable because there wasn’t enough RAM in the cluster. This clearly implied the cluster wasn’t big enough.\nLooking at the simple scaler shows it was ‘stuck’ at a populate.bash call, and wasn’t scaling up fast enough.\n\n\n11:35\nThe cluster is manually scaled up to 30 nodes:\ngcloud compute instance-groups managed resize gke-prod-highmem-pool-0df1a536-grp --size=30\nAt the same time, pods stuck in Pending state are deleted so they don’t become ghost pods, with:\nkubectl --namespace=datahub get pod | grep -v Running | grep -P 'm$' | awk '{print $1;}' | xargs -L1 kubectl --namespace=datahub delete pod\n\n\n11:40\nThe nodes have come up, so a populate.bash call is performed to pre-populate all user container images on the new nodes.\nUsers in Pending state are deleted again.\n\n\n11:46\nThe populate.bash call is complete, and everything is back online!"
   },
   {
-    "objectID": "incidents/2017-02-24-proxy-death-incident.html#conclusion",
-    "href": "incidents/2017-02-24-proxy-death-incident.html#conclusion",
-    "title": "Proxy eviction strands user",
+    "objectID": "incidents/2017-04-03-cluster-full-incident.html#conclusion",
+    "href": "incidents/2017-04-03-cluster-full-incident.html#conclusion",
+    "title": "Custom autoscaler does not scale up when it should",
     "section": "Conclusion",
-    "text": "Conclusion\nThis is almost a ‘perfect storm’ event. Three things colluded to make this outage happen:\n\nThe inodes issue, which causes containers to fail randomly\nThe fact that the proxy is a single point of failure with a longish recovery time in current JupyterHub architecture.\nKubeSpawner’s current design is inefficient at very high user volumes, and its request timeouts & other performance characteristics had not been tuned (because we have not needed to before).\n\nWe have both long term (~1-2 months) architectural fixes as well as short term tuning in place for all three of these issues."
+    "text": "Conclusion\nOur simple scaler didn’t scale up fast enough when a large number of students came back online quickly after a time of quiet (spring break). Took a while for this to get noticed, and manual scaling fixed everything."
   },
   {
-    "objectID": "incidents/2017-02-24-proxy-death-incident.html#action-items",
-    "href": "incidents/2017-02-24-proxy-death-incident.html#action-items",
-    "title": "Proxy eviction strands user",
+    "objectID": "incidents/2017-04-03-cluster-full-incident.html#action-items",
+    "href": "incidents/2017-04-03-cluster-full-incident.html#action-items",
+    "title": "Custom autoscaler does not scale up when it should",
     "section": "Action items",
-    "text": "Action items\n\nUpstream JupyterHub\n\nWork on abstracting the proxy interface, so the proxy is no longer a single point of failure. Issue\n\n\n\nUpstream KubeSpawner\n\nRe-architect the spawner to make a much smaller number of HTTP requests. DataHub has become big enough that this is a problem. Issue\nTune the HTTP client kubespawner uses. This would be an interim solution until (1) gets fixed. Issue\n\n\n\nDataHub configuration\n\nSet resource requests explicitly for hub and proxy, so they have less chance of getting evicted. Issue\nReduce the interval at which the hub checks to see if the proxy is running. PR\nSpeed up the fix for the inodes issue which is what triggered this whole issue."
+    "text": "Action items\n\nProcess\n\nWhen coming back from breaks, pre-scale the cluster back up.\nConsider cancelling spring break.\n\n\n\nMonitoring\n\nHave monitoring for pods stuck in non-Running states"
   },
   {
-    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html",
-    "href": "incidents/2017-03-06-helm-config-image-mismatch.html",
-    "title": "Non-matching hub image tags cause downtime",
+    "objectID": "incidents/2019-05-01-service-account-leak.html",
+    "href": "incidents/2019-05-01-service-account-leak.html",
+    "title": "Service Account key leak incident",
     "section": "",
-    "text": "On the evening of Mar 6, the hub on prod would not come up after an upgrade. The upgrade was to accommodate a new disk for cogneuro that had been tested on dev. After some investigation it was determined that the helm’s config did not match the hub’s image. After the hub image was rebuilt and pushed out, then tested on dev, it was pushed out to prod. The problem was fixed in about 40 minutes.\nA few days later (March 12), similar almost outage is avoided when -dev breaks and deployment is put on hold. More debugging shows the underlying cause is that git submodules are hard to use. More documentation is provided, and downtime is averted!"
+    "text": "Service account keys that granted restricted access to some of our cloud services were inadvertently leaked on GitHub. Google immediately notified us in seconds, and the credentials were revoked within the next few minutes."
   },
   {
-    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#summary",
-    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#summary",
-    "title": "Non-matching hub image tags cause downtime",
+    "objectID": "incidents/2019-05-01-service-account-leak.html#summary",
+    "href": "incidents/2019-05-01-service-account-leak.html#summary",
+    "title": "Service Account key leak incident",
     "section": "",
-    "text": "On the evening of Mar 6, the hub on prod would not come up after an upgrade. The upgrade was to accommodate a new disk for cogneuro that had been tested on dev. After some investigation it was determined that the helm’s config did not match the hub’s image. After the hub image was rebuilt and pushed out, then tested on dev, it was pushed out to prod. The problem was fixed in about 40 minutes.\nA few days later (March 12), similar almost outage is avoided when -dev breaks and deployment is put on hold. More debugging shows the underlying cause is that git submodules are hard to use. More documentation is provided, and downtime is averted!"
-  },
-  {
-    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#timeline",
-    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#timeline",
-    "title": "Non-matching hub image tags cause downtime",
-    "section": "Timeline",
-    "text": "Timeline\nAll times in PST\n\nMarch 6 2017 22:59\ndev changes are deployed but hub does not start correctly. The describe output for the hub shows repeated instances of:\nError syncing pod, skipping: failed to “StartContainer” for “hub-container” with CrashLoopBackOff: “Back-off 10s restarting failed container=hub-container pod=hub-deployment-3498421336-91gp3_datahub-dev(bfe7d8bd-0303-11e7-ade6-42010a80001a)\nhelm chart for -dev is deleted and reinstalled.\n\n\n23:11\ndev changes are deployed successfully and tested. cogneuro’s latest data is available.\n\n\n23:21\nChanges are deployed to prod. The hub does not start properly. get pod -o=yaml on the hub pod shows that the hub container has terminated. The hub log shows that it failed due to a bad configuration parameter.\n\n\n21:31\nWhile the helm chart had been updated from git recently, the latest tag for the hub did not correspond with the one in either prod.yaml or dev.yaml.\n\n\n21:41\nThe hub image is rebuilt and pushed out.\n\n\n21:45\nThe hub is deployed on -dev.\n\n\n21:46\nThe hub is tested on -dev then deployed on -prod.\n\n\n21:50\nThe hub is tested on -prod. Students are reporting that the hub had been down.\n\n\nMarch 12 19:57\nA new deploy is attempted on -dev, but runs into same error. Deployments are halted for more debugging this time, and more people are called on.\n\n\n23:21\nMore debugging reveals that the commit update looked like this:\ndiff --git a/chart b/chart\nindex e38aba2..c590340 160000\n--- a/chart\n+++ b/chart\n@@ -1 +1 @@\n-Subproject commit e38aba2c5601de30c01c6f3c5cad61a4bf0a1778\n+Subproject commit c59034032f8870d16daba7599407db7e6eb53e04\ndiff --git a/data8/dev.yaml b/data8/dev.yaml\nindex 2bda156..ee5987b 100644\n--- a/data8/dev.yaml\n+++ b/data8/dev.yaml\n@@ -13,7 +13,7 @@ publicIP: \"104.197.166.226\"\n\n singleuser:\n   image:\n-    tag: \"e4af695\"\n+    tag: \"1a6c6d8\"\n   mounts:\n     shared:\n       cogneuro88: \"cogneuro88-20170307-063643\"\nOnly the tag should’ve been the only thing updated. The chart submodule is updated to c59034032f8870d16daba7599407db7e6eb53e04, which is from February 25 (almost two weeks old). This is the cause of the hub failing, since it is using a really old chart commit with a new hub image."
+    "text": "Service account keys that granted restricted access to some of our cloud services were inadvertently leaked on GitHub. Google immediately notified us in seconds, and the credentials were revoked within the next few minutes."
   },
   {
-    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#section-8",
-    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#section-8",
-    "title": "Non-matching hub image tags cause downtime",
-    "section": "23:27",
-    "text": "23:27\nIt is determined that incomplete documentation about deployment processes caused git submodule update to be not run after a git pull, and so the chart was being accidentally moved back to older commits. Looking at the commit that caused the outage on March 6 showed the exact same root cause."
+    "objectID": "incidents/2019-05-01-service-account-leak.html#impact",
+    "href": "incidents/2019-05-01-service-account-leak.html#impact",
+    "title": "Service Account key leak incident",
+    "section": "Impact",
+    "text": "Impact\nDeployments are paused until this was fixed."
   },
   {
-    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#conclusion",
-    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#conclusion",
-    "title": "Non-matching hub image tags cause downtime",
-    "section": "Conclusion",
-    "text": "Conclusion\nGit submodules are hard to use, and break most people’s mental model of how git works. Since our deployment requires that the submodule by in sync with the images used, this caused an outage."
+    "objectID": "incidents/2019-05-01-service-account-leak.html#timeline",
+    "href": "incidents/2019-05-01-service-account-leak.html#timeline",
+    "title": "Service Account key leak incident",
+    "section": "Timeline",
+    "text": "Timeline\n\nMay 1 2019, 3:18 PM\nA template + documentation for creating new hubs easily is pushed to GitHub as a pull request. This inadvertently contained live credentials for pushing & pulling our (already public) docker images, and for access to our kubernetes clusters.\nGoogle immediately notified us via email within seconds that this might be a breach.\n\n\n3:19 PM\nDiscussion and notification starts in slack about dealing with the issue.\n\n\n3:27 PM\nBoth keys are revoked so they are no longer valid credentials.\n\n\n3:36 PM\nAll in-use resources are checked, and verified to not be compromised by automated bots looking for leaked accounts.\n\n\n3:40 PM\nAn email is sent out to all owners of the compromised project (ucb-datahub-2018) giving an all-clear."
   },
   {
-    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#action-items",
-    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#action-items",
-    "title": "Non-matching hub image tags cause downtime",
+    "objectID": "incidents/2019-05-01-service-account-leak.html#action-items",
+    "href": "incidents/2019-05-01-service-account-leak.html#action-items",
+    "title": "Service Account key leak incident",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nMake sure we treat any errors in -dev exactly like we would in prod. Any deployment error in prod should immediately halt future deployments & require a rollback or resolution before proceeding.\nWrite down actual deployment documentation & a checklist.\nMove away from git submodules to a separate versioned chart repository."
+    "text": "Action items\n\nDon’t duplicate service key credentials across multiple hubs. Issue\nSwitch to a different secret management strategy than what we have now. Issue"
   },
   {
-    "objectID": "incidents/index.html",
-    "href": "incidents/index.html",
-    "title": "Incident Reports",
+    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html",
+    "href": "incidents/2018-01-25-helm-chart-upgrade.html",
+    "title": "Accidental merge to prod brings things down",
     "section": "",
-    "text": "Order By\n       Default\n         \n          Date - Oldest\n        \n         \n          Date - Newest\n        \n         \n          Title\n        \n     \n  \n    \n      \n      \n    \n\n\n\n\n\nDate\n\n\nTitle\n\n\n\n\n\n\nFeb 9, 2017\n\n\nJupyterHub db manual overwrite\n\n\n\n\nFeb 24, 2017\n\n\nCustom Autoscaler gonee haywire\n\n\n\n\nFeb 24, 2017\n\n\nProxy eviction strands user\n\n\n\n\nMar 6, 2017\n\n\nNon-matching hub image tags cause downtime\n\n\n\n\nMar 20, 2017\n\n\nToo many volumes per disk leave students stuck\n\n\n\n\nMar 23, 2017\n\n\nWeird upstream ipython bug kills kernels\n\n\n\n\nApr 3, 2017\n\n\nCustom autoscaler does not scale up when it should\n\n\n\n\nMay 9, 2017\n\n\nOops we forgot to pay the bill\n\n\n\n\nOct 10, 2017\n\n\nDocker dies on a few Azure nodes\n\n\n\n\nOct 19, 2017\n\n\nBilling confusion with Azure portal causes summer hub to be lost\n\n\n\n\nJan 25, 2018\n\n\nAccidental merge to prod brings things down\n\n\n\n\nJan 26, 2018\n\n\nHub starts up very slow, causing outage for users\n\n\n\n\nFeb 6, 2018\n\n\nAzure PD refuses to detach, causing downtime for data100\n\n\n\n\nFeb 28, 2018\n\n\nA node hangs, causing a subset of users to report issues\n\n\n\n\nJun 11, 2018\n\n\nAzure billing issue causes downtime\n\n\n\n\nFeb 25, 2019\n\n\nAzure Kubernetes API Server outage causes downtime\n\n\n\n\nMay 1, 2019\n\n\nService Account key leak incident\n\n\n\n\nJan 20, 2022\n\n\nHubs throwing 505 errors\n\n\n\n\nFeb 1, 2024\n\n\nCore nodes being autoscaled, configurable HTTP proxy crashes\n\n\n\n\n\nNo matching items",
-    "crumbs": [
-      "Contributing",
-      "Incident Reports"
-    ]
+    "text": "On January 25, 2018, a new version of the helm chart was installed on the staging hub. It was not immediately merged to production because there were active labs throughout the day. While preparing another course’s hub via Travis CI, the Data 8 change was accidentally merged from staging to production. This production hub went down because the new helm chart’s jupyterhub image was broken."
   },
   {
-    "objectID": "incidents/2017-03-20-too-many-volumes.html",
-    "href": "incidents/2017-03-20-too-many-volumes.html",
-    "title": "Too many volumes per disk leave students stuck",
+    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#summary",
+    "href": "incidents/2018-01-25-helm-chart-upgrade.html#summary",
+    "title": "Accidental merge to prod brings things down",
     "section": "",
-    "text": "From sometime early March 20 2017 till about 1300, some new student servers were stuck in Pending forever, giving them 500 errors. This was an unintended side-effect of reducing student memory limit to 1G while keeping the size of our nodes constant, causing us to hit a Google Cloud limit on number of disks per node. This was fixed by spawning more nodes that were smaller."
+    "text": "On January 25, 2018, a new version of the helm chart was installed on the staging hub. It was not immediately merged to production because there were active labs throughout the day. While preparing another course’s hub via Travis CI, the Data 8 change was accidentally merged from staging to production. This production hub went down because the new helm chart’s jupyterhub image was broken."
   },
   {
-    "objectID": "incidents/2017-03-20-too-many-volumes.html#summary",
-    "href": "incidents/2017-03-20-too-many-volumes.html#summary",
-    "title": "Too many volumes per disk leave students stuck",
-    "section": "",
-    "text": "From sometime early March 20 2017 till about 1300, some new student servers were stuck in Pending forever, giving them 500 errors. This was an unintended side-effect of reducing student memory limit to 1G while keeping the size of our nodes constant, causing us to hit a Google Cloud limit on number of disks per node. This was fixed by spawning more nodes that were smaller."
+    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#timeline",
+    "href": "incidents/2018-01-25-helm-chart-upgrade.html#timeline",
+    "title": "Accidental merge to prod brings things down",
+    "section": "Timeline",
+    "text": "Timeline\n\n2018-01-25 14:30\nThe helm chart for datahub was upgraded to a beta of v0.6 to make use of a new image puller. This was merged into the staging branch. After some initial debugging, the helm chart was installed successfully and the image puller worked correctly. However, the staging hub was not tested.\nSince labs were scheduled throughout the day until 7p, it was decided to delay the upgrade of the production hub until after 7p.\n\n\n15:30\nWhile a different hub was being managed in Travis CI, the production hub for Data 8 was accidentally upgraded. This upgrade brought with it the faulty hub image from staging which wasn’t working.\n\n\n16:11\nGSIs report in slack that the hub is down for lab users. It is confirmed that the hub process has crashed due to a shared C library included from a python library. It is decided that the quickest way to bring the hub back up is to downgrade the helm-chart back to v0.5.0.\n\n\n16:35\nThe chart is installed into the staging repo, merged to staging, and checked on the staging hub. It is then merged into production and brought online there."
   },
   {
-    "objectID": "incidents/2017-03-20-too-many-volumes.html#timeline",
-    "href": "incidents/2017-03-20-too-many-volumes.html#timeline",
-    "title": "Too many volumes per disk leave students stuck",
-    "section": "Timeline",
-    "text": "Timeline\n\nMarch 18, 16:30\nRAM per student is reduced from 2G to 1G, as a resource optimization measure. The size of our nodes remains the same (26G RAM), and many are cordonned off and slowly decommissioned over the coming few days.\nLife seems fine, given the circumstances.\n\n\nMarch 20, 12:44\nNew student servers report a 500 error preventing them from logging on. This is deemed widespread & not an isolated incident.\n\n\n12:53\nA kubectl describe pod on an affected student’s pod shows it’s stuck in Pending state, with an error message:\npod failed to fit in any node fit failure on node (XX): MaxVolumeCount\nThis seems to be common problem for all the new student servers, which are all stuck in Pending state.\nGoogling leads to https://github.com/kubernetes/kubernetes/issues/24317 - even though Google Compute Engine can handle more than 16 disks per node (we had checked this before deploying), Kubernetes itself still can not. This wasn’t foreseen, and seemed to be the direct cause of the incident.\n\n\n13:03\nA copy of the instance template that is used by Google Container Engine is made and then modified to spawn smaller nodes (n1-highmem-2 rather than n1-highmem-4). The managed instance group used by Google Container Engine is then modified to use the new template. This was the easiest way to not distrupt students for whom things are working, while also allowing new students to be able to log in.\nThis new instance group was then set to expand for 30 new nodes, which will provide capacity for about 12 students each. populate.bash was also run to make sure that students pods start up on time in the newnodes.\n\n\n13:04\nThe simple autoscaler is stopped, on fear that it’ll be confused by the unusual mixed state of the nodes and do something wonky.\n\n\n13:11\nAll the new nodes are online, and populate.bash has completed. Pods start leaving the Pending state.\nHowever, since it’s been more than the specified timeout that JupyterHub will wait before giving up on Pod (5 minutes), JupyterHub doesn’t know the pods exist. This causes state of cluster + state in JupyterHub to go out of sync, causing the dreaded ‘redirected too many times’ error. Admins need to manually stop and start user pods in the control panel as users report this to fix this issue.\n\n\n14:23\nThe hub and proxy pods are restarted since there were plenty of ‘redirected too many times’ errors. This seems to catch most users state, although some requests still failed with a 599 timeout (similar to an earlier incident, but much less frequent). A long tail of manual user restarts are performed by admins over the next few days."
+    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#conclusion",
+    "href": "incidents/2018-01-25-helm-chart-upgrade.html#conclusion",
+    "title": "Accidental merge to prod brings things down",
+    "section": "Conclusion",
+    "text": "Conclusion\nA relatively large change was made to the hub configuration with insufficient testing on the staging server. This was compounded when the change was accidentally merged to production."
   },
   {
-    "objectID": "incidents/2017-03-20-too-many-volumes.html#action-items",
-    "href": "incidents/2017-03-20-too-many-volumes.html#action-items",
-    "title": "Too many volumes per disk leave students stuck",
-    "section": "Action Items",
-    "text": "Action Items\n\nUpstream: Kubernetes\n\nKeep an eye on the status of the bug we ran into\n\n\n\nUpstream: JupyterHub\n\nTrack down and fix the ‘too many redirects’ issue at source. Issue\n\n\n\nCleanup\n\nDelete all the older larger nodes that are no longer in use. (Done!)\n\n\n\nMonitoring\n\nHave alerting for when there are any number of pods in Pending state for a non-negligible amount of time. There is always something wrong when this happens."
+    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#action-items",
+    "href": "incidents/2018-01-25-helm-chart-upgrade.html#action-items",
+    "title": "Accidental merge to prod brings things down",
+    "section": "Action items",
+    "text": "Action items\n\nProcess\n\nAdmins should refamiliarize themselves with the deployment policy to check the staging hub before changes are merged to production.\nDetermine if there is a way to block merges to production if the staging hub is not online.\nDetermine if there is a way to contextualize the Travis CI interface so that it is obvious which deployment is being managed."
   },
   {
-    "objectID": "incidents/2017-10-10-hung-nodes.html",
-    "href": "incidents/2017-10-10-hung-nodes.html",
-    "title": "Docker dies on a few Azure nodes",
+    "objectID": "incidents/2018-01-26-hub-slow-startup.html",
+    "href": "incidents/2018-01-26-hub-slow-startup.html",
+    "title": "Hub starts up very slow, causing outage for users",
     "section": "",
-    "text": "On Oct 10, 2017, some user pods were not starting or terminating correctly. After checking node status, it was found that all affected pods were running on two specific nodes. The docker daemon wasn’t responsive on these nodes so they were cordoned off. User pods were then able to start correctly."
+    "text": "On January 26, 2018, a new version of the helm chart was being installed on the production hub. Though the pod prepuller worked fine on the staging cluster, the prepuller never successfully finished on prod. This caused the CI to error because helm ran for too long. Additionally, the hub was taking a very long time to check user routes. After users were deleted in the hub’s orm and the hub was restarted, it came back up fairly quickly."
   },
   {
-    "objectID": "incidents/2017-10-10-hung-nodes.html#summary",
-    "href": "incidents/2017-10-10-hung-nodes.html#summary",
-    "title": "Docker dies on a few Azure nodes",
+    "objectID": "incidents/2018-01-26-hub-slow-startup.html#summary",
+    "href": "incidents/2018-01-26-hub-slow-startup.html#summary",
+    "title": "Hub starts up very slow, causing outage for users",
     "section": "",
-    "text": "On Oct 10, 2017, some user pods were not starting or terminating correctly. After checking node status, it was found that all affected pods were running on two specific nodes. The docker daemon wasn’t responsive on these nodes so they were cordoned off. User pods were then able to start correctly."
+    "text": "On January 26, 2018, a new version of the helm chart was being installed on the production hub. Though the pod prepuller worked fine on the staging cluster, the prepuller never successfully finished on prod. This caused the CI to error because helm ran for too long. Additionally, the hub was taking a very long time to check user routes. After users were deleted in the hub’s orm and the hub was restarted, it came back up fairly quickly."
   },
   {
-    "objectID": "incidents/2017-10-10-hung-nodes.html#timeline",
-    "href": "incidents/2017-10-10-hung-nodes.html#timeline",
-    "title": "Docker dies on a few Azure nodes",
+    "objectID": "incidents/2018-01-26-hub-slow-startup.html#timeline",
+    "href": "incidents/2018-01-26-hub-slow-startup.html#timeline",
+    "title": "Hub starts up very slow, causing outage for users",
     "section": "Timeline",
-    "text": "Timeline\n\n2017-05-09 10:45a\nA report in the course Piazza said that two students couldn’t start their servers. The /hub/admin interface was not able to start them either. It was reported that the students may have run out of memory.\n\n\n12:29p\nThe user pods were stuck in Terminating state and would not respond to explicit delete. The pods were forcefully deleted with kubectl --namespace=prod delete pod jupyter-&lt;name&gt; --grace-period=0 --force. The user pods started correctly via /hub/admin.\n\n\n13:27\nIt was reported in the course slack that another student’s server wasn’t starting correctly. After checking one of the pod logs, it was observed that the node hosting the pods, k8s-pool1-19522833-13, was also hosting many more pods stuck in a Terminating state. docker ps was hanging on that node. The node was cordoned.\n\n\n13:42\nIt was reported in slack that the student’s server was able to start.\nBy this time, the cluster was checked for all pods to see if any other nodes were hosting an unusual number of pods in Terminating. It was found that k8s-pool2-19522833-9 was in a similar state. All stuck pods on that node were forcefully deleted and the node was also cordoned. docker ps was hung on that node too. pool2-…-9 had a load of 530 while pool1-…-13 had a load of 476. On the latter, hypercube was at 766% cpu utilization while it was nominal on the former. Node pool1-…-13 was rebooted from the shell however it did not come back online. The node was manually restarted from the Azure portal but it still didn’t come back.\nA node previously cordoned on another day, pool1-…-14, was rebooted. It came back online and was uncordoned.\n\n\n13:51\nSome relevant systemctl status docker logs were captured from pool2-…-9:\nOct 10 20:55:30 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:30.790401257Z\" level=error msg=\"containerd: start container\" error=\"containerd: container did not start before the specified timeout\" id=abd267ef08b4a4184e19307be784d62470f9a713b59e406249c6cdf0bb333260\nOct 10 20:55:30 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:30.790923460Z\" level=error msg=\"Create container failed with error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:30 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:30.810309575Z\" level=error msg=\"Handler for POST /v1.24/containers/abd267ef08b4a4184e19307be784d62470f9a713b59e406249c6cdf0bb333260/start returned error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.146453953Z\" level=error msg=\"containerd: start container\" error=\"containerd: container did not start before the specified timeout\" id=2ba6787503ab6123b509811fa44c7e42986de0b800cc4226e2ab9484f54e8741\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.147565759Z\" level=error msg=\"Create container failed with error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.166295370Z\" level=error msg=\"Handler for POST /v1.24/containers/2ba6787503ab6123b509811fa44c7e42986de0b800cc4226e2ab9484f54e8741/start returned error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.169360588Z\" level=error msg=\"Handler for GET /v1.24/containers/json returned error: write unix /var/run/docker.sock-&gt;@: write: broken pipe\"\nOct 10 20:55:36 k8s-pool2-19522833-9 dockerd[1237]: http: multiple response.WriteHeader calls\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.280209444Z\" level=error msg=\"Handler for GET /v1.24/containers/610451d9d86a58117830ea7c0189f6157ba9a9602739ee23723e923de8c7e23e/json returned error: No such container: 610451d9d86a58117830ea7c0189f6157ba9a9602739ee23723e923de8c7e23e\"\nOct 10 20:55:39 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:39.095888009Z\" level=error msg=\"Handler for GET /v1.24/containers/54b64ca3c1e7ef4a04192ccdaf1cb9309d73acebd7a08e13301f3263de3d376a/json returned error: No such container: 54b64ca3c1e7ef4a04192ccdaf1cb9309d73acebd7a08e13301f3263de3d376a\"\n\n\n14:00\ndatahub@k8s-pool2-19522833-9:~$ ps aux | grep exe | wc -l\n520\ndatahub@k8s-pool2-19522833-9:~$ ps aux | grep exe | head -5\nroot        329  0.0  0.0 126772  9812 ?        Dsl  00:36   0:00 /proc/self/exe init\nroot        405  0.0  0.0  61492  8036 ?        Dsl  00:36   0:00 /proc/self/exe init\nroot        530  0.0  0.0 127028  8120 ?        Dsl  00:36   0:00 /proc/self/exe init\nroot        647  0.0  0.0 127028  8124 ?        Dsl  13:07   0:00 /proc/self/exe init\nroot        973  0.0  0.0  77884  8036 ?        Dsl  13:10   0:00 /proc/self/exe init\n\n\n14:30\npool1-…-13 was manually stopped in the Azure portal, then manually started. It came back online afterwards and docker was responsive. It was uncordoned.\npool2-…-9 was manually stopped in the Azure portal.\n\n\n14:45\npool2-…-9 completed stopping and was manually started in the Azure portal.\n\n\n17:25\nIt was observed that /var/lib/docker on pool1-19522833-13/10.240.0.7 was on / (sda) and not on /mnt (sdb)."
+    "text": "Timeline\n\n2018-01-26 15:00\nThe helm chart for datahub was upgraded to a beta of v0.6 to make use of a new image puller. This was merged into the staging branch, successfully tested on the staging, and passed CI checks on prod. It was then merged to prod.\n\n\n15:15\nhelm times out because the prepuller never completes. It is determined that the master node on staging is cordoned while the master node on prod is not and has:\ntaints:\n  - effect: NoSchedule\n    key: node-role.kubernetes.io/master\n    timeAdded: null\n    value: \"true\"\nThe master is cordoned on prod and a new build is started in CI.\n\n\n15:33\nAfter CI times out again due to the prepuller, it is discovered that the master node has been uncordoned. Though the hub and proxy pods restart, the hub is taking a very long time to check user routes. It is slower than the most recent hub restart which was itself slow enough to warrant a new issue on jupyterhub, https://github.com/jupyterhub/jupyterhub/issues/1633.\n\n\n13:40\nIt is decided that the most expedient way to get the hub up is to delete users from the orm.\n\n\n13:50\nThe following command is run after the database is backed up:\ndelete from users where users.id in (select users.id from users join spawners on spawners.user_id = users.id where server_id is null);\ndeleting 4902 records. The hub pod is deleted and the hub comes up shortly after."
   },
   {
-    "objectID": "incidents/2017-10-10-hung-nodes.html#conclusion",
-    "href": "incidents/2017-10-10-hung-nodes.html#conclusion",
-    "title": "Docker dies on a few Azure nodes",
+    "objectID": "incidents/2018-01-26-hub-slow-startup.html#conclusion",
+    "href": "incidents/2018-01-26-hub-slow-startup.html#conclusion",
+    "title": "Hub starts up very slow, causing outage for users",
     "section": "Conclusion",
-    "text": "Conclusion\nDocker was hung on two nodes, preventing pods from starting or stopping correctly."
+    "text": "Conclusion\nAt 5000, the hub takes long enough to restart to inconvenience the number of active users at any one time."
   },
   {
-    "objectID": "incidents/2017-10-10-hung-nodes.html#action-items",
-    "href": "incidents/2017-10-10-hung-nodes.html#action-items",
-    "title": "Docker dies on a few Azure nodes",
+    "objectID": "incidents/2018-01-26-hub-slow-startup.html#action-items",
+    "href": "incidents/2018-01-26-hub-slow-startup.html#action-items",
+    "title": "Hub starts up very slow, causing outage for users",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nWhen there are multiple reports of student servers not starting or stopping correctly, check to see if the user pods were run on the same node(s).\nDetermine how many nodes are not mounting /var/lib/docker on sdb1.\n\n\n\nMonitoring\n\nLook for elevated counts of pods stuck in Terminating state. For example, kubectl --namespace=prod get pod -o wide| grep Terminating"
+    "text": "Action items\n\nProcess\n\nThe prepuller should be fixed so that helm does not time out.\nThe hub route checking should be parallelized so that startup is not slow.\nThe staging hub should be seeded with users so that scaling issues can be exposed prior to reaching production."
   },
   {
-    "objectID": "incidents/2017-02-24-autoscaler-incident.html",
-    "href": "incidents/2017-02-24-autoscaler-incident.html",
-    "title": "Custom Autoscaler gonee haywire",
+    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html",
+    "href": "incidents/2017-03-23-kernel-deaths-incident.html",
+    "title": "Weird upstream ipython bug kills kernels",
     "section": "",
-    "text": "On the evening of February 24, 2017, a premature version of the Autoscaler script for the Datahub deployment was mistakenly run on the prod cluster, resulting in a large amount of nodes (roughly 30-40) being set as unschedulable for about 20 minutes. Though no information was lost nor service critically disturbed, it was necessary to manually re-enable these nodes to be scheduled."
+    "text": "A seemingly unrelated change caused user kernels to die on start (making notebook execution impossible) for newly started user servers from about Mar 22 19:30 to Mar 23 09:45. Most users didn’t see any errors until start of class at about 9AM, since they were running servers that were previously started."
   },
   {
-    "objectID": "incidents/2017-02-24-autoscaler-incident.html#summary",
-    "href": "incidents/2017-02-24-autoscaler-incident.html#summary",
-    "title": "Custom Autoscaler gonee haywire",
+    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#summary",
+    "href": "incidents/2017-03-23-kernel-deaths-incident.html#summary",
+    "title": "Weird upstream ipython bug kills kernels",
     "section": "",
-    "text": "On the evening of February 24, 2017, a premature version of the Autoscaler script for the Datahub deployment was mistakenly run on the prod cluster, resulting in a large amount of nodes (roughly 30-40) being set as unschedulable for about 20 minutes. Though no information was lost nor service critically disturbed, it was necessary to manually re-enable these nodes to be scheduled."
+    "text": "A seemingly unrelated change caused user kernels to die on start (making notebook execution impossible) for newly started user servers from about Mar 22 19:30 to Mar 23 09:45. Most users didn’t see any errors until start of class at about 9AM, since they were running servers that were previously started."
   },
   {
-    "objectID": "incidents/2017-02-24-autoscaler-incident.html#timeline",
-    "href": "incidents/2017-02-24-autoscaler-incident.html#timeline",
-    "title": "Custom Autoscaler gonee haywire",
+    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#timeline",
+    "href": "incidents/2017-03-23-kernel-deaths-incident.html#timeline",
+    "title": "Weird upstream ipython bug kills kernels",
     "section": "Timeline",
-    "text": "Timeline\nAs of this commit in the Autoscaler branch history, there exists a scale.py file that would based on the utilization of the cluster, mark a certain number of nodes unschedulable before attempting to shut down nodes with no pods in them. Unfortunately, this script was executed prematurely, and without configuration, looked to execute in whatever context currently specified in .kube/config, which ended up being the production cluster rather than the dev cluster.\n\n2017-02-24 11:14 PM\nScript is mistakenly executed. A bug in the calculations for the utilization of the cluster leads to about 40 nodes being marked as unschedulable. The mistake is noted immediately.\n\n\n2017-02-24 11:26 PM\nThe unschedulability of these nodes is reverted. All nodes in the cluster were first all set to be schedulable to ensure that no students current and future would be disturbed. Immediately after, 10 of the most idle nodes on the cluster were manually set to be unschedulable (to facilitate them later being manually descaled - to deal with https://github.com/data-8/infrastructure/issues/6) using kubectl cordon &lt;node_name&gt;."
+    "text": "Timeline\n\nMarch 22, around 19:30\nA deployment is performed, finally deploying https://github.com/data-8/jupyterhub-k8s/pull/146 to production. It seemed to work fine on -dev, and on prod as well. However, the testing regimen was only to see if a notebook server would show up - not if a kernel would spawn.\n\n\nMar 23, 09:08\nStudents report that their kernels keep dying. This is confirmed to be a problem for all newly launched notebooks, in both prod and dev.\n\n\n09:16\nThe last change to the repo (an update of the single-user image) is reverted, to check if that was causing the problem. This does not improve the situation. Debugging continues, but with no obvious angles of attack.\n\n\n09:41\nAfter debugging produces no obvious culprits, the state of the entire infrastructure for prod is reverted to a known good state from a few days ago. This was done with:\n./deploy.py prod data8 25abea764121953538713134e8a08e0291813834\n25abea764121953538713134e8a08e0291813834 is the commit hash of a known good commit from March 19. Our disciplined adherence to immutable & reproducible deployment paid off, and we were able to restore new servers to working order with this!\nStudents are now able to resume working after a server restart. A mass restart is also performed to aid this.\nDev is left in a broken state in an attempt to debug.\n\n\n09:48\nA core Jupyter Notebook dev at BIDS attempts to debug the problem, since it seems to be with the notebook itself and not with JupyterHub.\n\n\n11:08\nCore Jupyter Notebook dev confirms that this makes no sense.\n\n\n14:55\nAttempts to isolate the bug start again, mostly by using git bisect to deploy different versions of our infrastructure to dev until we find what broke.\n\n\n15:30\nhttps://github.com/data-8/jupyterhub-k8s/pull/146 is identified as the culprit. It continues to not make sense.\n\n\n17:25\nA very involved and laborious revert of the offending part of the patch is done in https://github.com/jupyterhub/kubespawner/pull/37. Core Jupyter Notebook dev continues to confirm this makes no sense.\nhttps://github.com/data-8/jupyterhub-k8s/pull/152 is also merged, and deployed shortly after verifying that everything (including starting kernels & executing code) works fine on dev. Deployed to prod and everything is fine."
   },
   {
-    "objectID": "incidents/2017-02-24-autoscaler-incident.html#conclusion",
-    "href": "incidents/2017-02-24-autoscaler-incident.html#conclusion",
-    "title": "Custom Autoscaler gonee haywire",
+    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#conclusion",
+    "href": "incidents/2017-03-23-kernel-deaths-incident.html#conclusion",
+    "title": "Weird upstream ipython bug kills kernels",
     "section": "Conclusion",
-    "text": "Conclusion\nA cluster autoscaler script was accidentally run against the production cluster instead of the dev cluster, reducing capacity for new user logins for about 12 minutes. There was still enough capacity so we had no adverse effects."
+    "text": "Conclusion\nInsufficient testing procedures caused a new kind of outage (kernel dying) that we had not seen before. However, since our infrastructure was immutable & reproducible, our outage really only lasted about 40 minutes (from start of lab when students were starting containers until the revert). Deeper debugging produced a fix, but attempts to understand why the fix works are ongoing.\nUpdate: We have found and fixed the underlying issue"
   },
   {
-    "objectID": "incidents/2017-02-24-autoscaler-incident.html#action-items",
-    "href": "incidents/2017-02-24-autoscaler-incident.html#action-items",
-    "title": "Custom Autoscaler gonee haywire",
-    "section": "Action Items",
-    "text": "Action Items\n\nDatahub Deployment Changes\n\nThe Autoscaler should not be run unless the context is explicitly set via environment variables or command line arguments. This is noted in the comments of the pull request for the Autoscaler.\nThe idea of the ‘current context’ should be abolished in all the tools we build / read.\n\n\n\nFuture organizational change\n\nUse a separate billing account for production vs development clusters. This makes it harder to accidentally run things on the wrong cluster"
+    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#action-items",
+    "href": "incidents/2017-03-23-kernel-deaths-incident.html#action-items",
+    "title": "Weird upstream ipython bug kills kernels",
+    "section": "Action items",
+    "text": "Action items\n\nProcess\n\nDocument and formalize the testing process for post-deployment checks.\nSet a short timeout (maybe ten minutes?) after which investigation temporarily stops and we revert our deployment to a known good state.\n\n\n\nUpstream KubeSpawner\n\nContinue investigating https://github.com/jupyterhub/kubespawner/issues/31, which was the core issue that prompted the changes that eventually led to the outage."
   },
   {
-    "objectID": "incidents/2018-02-28-hung-node.html",
-    "href": "incidents/2018-02-28-hung-node.html",
-    "title": "A node hangs, causing a subset of users to report issues",
+    "objectID": "incidents/2017-02-09-datahub-db-outage.html",
+    "href": "incidents/2017-02-09-datahub-db-outage.html",
+    "title": "JupyterHub db manual overwrite",
     "section": "",
-    "text": "On February 28, 2018, a handful of users reported on piazza that there servers wouldn’t start. It was determined that all problematic servers were running on the same node. After the node was cordoned and rebooted, the student servers were able to start properly."
+    "text": "Datahub was reportedly down at 1am. Users attempting to log in to datahub were greeted with a proxy error. The hub pod was up but the log was full of sqlite errors. After the hub pod was deleted and a new one came up, students logging in to datahub found their notebooks were missing and their home directories were empty. Once this was fixed, some students still were being logged in as a different particular user. Finally, students with a ‘.’ in their username were still having issues after everyone else was fine. This was all fixed and an all-clear signalled at about 2017-02-09 11:35 AM."
   },
   {
-    "objectID": "incidents/2018-02-28-hung-node.html#summary",
-    "href": "incidents/2018-02-28-hung-node.html#summary",
-    "title": "A node hangs, causing a subset of users to report issues",
+    "objectID": "incidents/2017-02-09-datahub-db-outage.html#summary",
+    "href": "incidents/2017-02-09-datahub-db-outage.html#summary",
+    "title": "JupyterHub db manual overwrite",
     "section": "",
-    "text": "On February 28, 2018, a handful of users reported on piazza that there servers wouldn’t start. It was determined that all problematic servers were running on the same node. After the node was cordoned and rebooted, the student servers were able to start properly."
+    "text": "Datahub was reportedly down at 1am. Users attempting to log in to datahub were greeted with a proxy error. The hub pod was up but the log was full of sqlite errors. After the hub pod was deleted and a new one came up, students logging in to datahub found their notebooks were missing and their home directories were empty. Once this was fixed, some students still were being logged in as a different particular user. Finally, students with a ‘.’ in their username were still having issues after everyone else was fine. This was all fixed and an all-clear signalled at about 2017-02-09 11:35 AM."
   },
   {
-    "objectID": "incidents/2018-02-28-hung-node.html#timeline",
-    "href": "incidents/2018-02-28-hung-node.html#timeline",
-    "title": "A node hangs, causing a subset of users to report issues",
+    "objectID": "incidents/2017-02-09-datahub-db-outage.html#timeline",
+    "href": "incidents/2017-02-09-datahub-db-outage.html#timeline",
+    "title": "JupyterHub db manual overwrite",
     "section": "Timeline",
-    "text": "Timeline\n\n2018-02-28 21:21\nThree students report problems starting their server on piazza and a GSI links to the reports on slack. More reports come in by 21:27.\n\n\n21:30\nThe infrastructure team is alerted to the problem. The command kubectl --namespace=prod get pod -o wide | egrep -v -e prepull -e Running shows that all non-running pods were scheduled on the same node. Most of the pods have an “Unknown” status while the rest are in “Terminating”. The oldest problematic pod is 29m.\n\n\n21:34\nThe node k8s-pool1-19522833-9 is cordoned. It has a load of about 90 with no processes consuming much CPU. The node is rebooted via sysrq trigger. The hung pods remain stuck.\n\n\n21:39\nWhen the node comes back online, kubectl reports no more hung pods. Students are able to start their servers."
-  },
-  {
-    "objectID": "incidents/2018-02-28-hung-node.html#conclusion",
-    "href": "incidents/2018-02-28-hung-node.html#conclusion",
-    "title": "A node hangs, causing a subset of users to report issues",
-    "section": "Conclusion",
-    "text": "Conclusion\nA problematic VM prevented nodes from launching pods. Once the VM was cordoned and rebooted, pods launch without trouble."
+    "text": "Timeline\n\n2017-02-09 00:25 - 00:29 AM\nAttempting to debug some earlier 400 errors, Trying to set base_url and ip to something incorrect to see if it will cause a problem.\nkubectl exec hub-deployment-something --namespace=datahub -it bash\napt-get install sqlite3\nsqlite3\nATTACH 'jupyterhub.sqlite AS my_db;\nSELECT name FROM my_db.sqlite_master WHERE type='table';\nSELECT * FROM servers;\nSELECT * FROM servers WHERE base_url LIKE '%&lt;USER&gt;%';\nUPDATE servers SET ip='' WHERE base_url LIKE '%&lt;USER&gt;%';\nUPDATE servers SET base_url='/&lt;something-wrong&gt; WHERE base_url LIKE '%&lt;USER&gt;%';\nCtrl+D (exit back into bash shell)\nchecked datahub.berkeley.edu, and nothing happened to the account saw that the sql db was not updated, attempt to run .save\n```bash\nsqlite3\n.save jupyterhub.sqlite\nThis replaced the db with an empty one, since ATTACH was not run beforehand.\n\n\n0:25:59 AM\nFollowing exception shows up in hub logs:\nsqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: proxies [SQL: 'SELECT proxies.id AS proxies_id, proxies._public_server_id AS proxies__public_server_id, proxies._api_server_id AS proxies__api_server_id \\nFROM proxies \\nWHERE proxies.id = ?'] [parameters: (1,)]\nThis continues for hub table as well, since those two seem to be most frequently used.\n\n\n1:12 AM\nSam’s roommate notices that he can log in to datahub but all his notebooks are gone. We notice that there are only ~50 users on the JHub admin panel when there used to be ~1000, so we believe that this is because the JHub sqlite user database got wiped/corrupted, then created an account for his roommate when he logged in, then created a new persistent disk since it lost track of his old one.\nThis is confirmed soon after:\n$ kubectl --namespace=datahub get pvc | grep  &lt;username&gt;\nclaim-&lt;username&gt;-257     Bound     pvc-3b405e13-ddb4-11e6-98ef-42010af000c3   10Gi       RWO           21d\nclaim-&lt;username&gt;-51      Bound     pvc-643dd900-eea7-11e6-a291-42010af000c3   10Gi       RWO           5m\n\n\n1:28 AM\nWe shut down the hub pod by scaling the replicas to 0.\nWe then begin recreating the JHub sqlite database by taking the Kubernetes PVCs and matching them back with the user ids. We could do this because the name of the PVC contains a sanitized form of the username and the userid.\nHere’s the notebook that was used to recreate the db from PVCs: pvc-sqlite.ipynb 2017-02-09-datahub-db-outage-pvc-recreate-script.ipynb\n\n\n2:34 AM\nWe recreate the sqlite3 database. Initially each user’s cookie_id was set to a dummy cookie value.\n\n\n2:42 AM\nUser cookie_id values are changed to null rather than dummy value. The sqlite file is then attached back to datahub. The number of users shown on admin page is back to ~1000. The hub was up, and a spot check of starting other user’s servers seem to work. Some users get redirected to one particular user, but deleting and recreating the affected user seems to fix this.\n\n\n10:11 AM\nAttempt to log everyone out by changing cookie secret in hub pod at /srv/jupyterhub/jupyterhub_cookie_secret. Just one character near the end was changed, and pod restarted. No effect. One character at the beginning of secret was changed next, and restarted - this caused actual change, and logged all users out.\nPeople are still being redirected to one particular user’s account when they log in. More looking around required.\n\n\n10:17 AM\nJohn Denero advises students to use ds8.berkeley.edu right now. ds8.berkeley.edu promptly starts crashing because it does not have resources for a data8 level class.\n\n\n10:29 AM\nAll user pods are deleted, which finally properly logs everyone out. However, people logging in are still all getting the same user’s pods.\n\n\n10:36 AM\nNotice that cookie_id column in the user database table is empty for many users, and the user that everyone is being logged in as has an empty cookie_id too and is the ‘first’ on the table when sorted in ascending by id. Looking at the JupyterHub code, cookie_id is always supposed to be set to a uuid, and never supposed to be empty. Setting cookie_id for users fixes their issues, and seems to spawn them into their own notebook.\n\n\n10:45 AM\nA script is run that populates cookie_id for all users, and restarts the hub to make sure there’s no stale cache in RAM. All user pods are deleted again. Most users are back online now! More users start testing and confirming things are working for them.\n\n\n10:53 AM\nUser with a ‘.’ in their name reports that they’re getting an empty home directory. More investigation shows two users - one with a ‘.’ in their name that is newer, and one with a ‘-’ in their name instead of ‘.’ that is older. Hypothesis is that one of them is the ‘original’, but they’re all attaching to a new one that is empty. Looking at pvcs confirms this - there are two PVCs for users with a . in their name who have tried to log in, and they differ only by ids.\nThere is some confusion about users ending up on prob140, because the data8.org homework link is changed to use that temporarily.\n\n\n11:05 AM\nDirectly modifying the user table to rename the user with the ‘-’ in the name to have a ‘.’ seems to work for people.\n\n\n11:15 AM\nA script is run that modifies the database user table for all users with a ‘-’ in their name, and the ‘-’ is replaced with a ‘.’. The new users created with the ‘.’ in their name are dropped before this.\n\n\n11:17 AM\nAll clear given for datahub.berkeley.edu\n\n\n11:19 AM\nLocally verified that running .save  on sqlite3 will overwrite the db file without any confirmation, and is most likely cause of the issue. Conclusion Accidental overwriting of the sqlite file during routine debugging operation led all tables being deleted. Users were getting new user ids when they were logging in now, causing them to get new disks provisioned - and these disks were empty. During reconstruction of the db, cookie_id was missing for several users, causing them all to log in to one particular user’s notebook. Users with ‘.’ in their name were also set up slightly incorrectly - their pods have ‘-’ in them but the user name should have a ‘.’."
   },
   {
-    "objectID": "incidents/2018-02-28-hung-node.html#action-items",
-    "href": "incidents/2018-02-28-hung-node.html#action-items",
-    "title": "A node hangs, causing a subset of users to report issues",
+    "objectID": "incidents/2017-02-09-datahub-db-outage.html#action-items",
+    "href": "incidents/2017-02-09-datahub-db-outage.html#action-items",
+    "title": "JupyterHub db manual overwrite",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nMonitor the cluster for non-running pods and send an alert if the count exceeds a threshold or if the non-running pods are clustered on the same node(s)."
+    "text": "Action items\n\nUpstream bug reports for JupyterHub\n\nJupyterHub only uses a certain length of the cookie secret, and discards the rest. This causes confusion when trying to change it to log people out. Issue\nThe cookie_id column in the users table should have UNIQUE and NOT NULL constraints. Issue\n\n\n\nUpstream bug reports for KubeSpawner\n\nSupport using username hashes in PVC and Pod Names rather than user ids, so that pod and PVC names remain constant even when DB is deleted. Issue\n\n\n\nUpstream bug reports for OAuthenticator\n\nSupport setting id of user in user table to be same as ‘id’ provided by Google authenticator, thus providing a stable userid regardless of when the user first logged in. Issue\n\n\n\nDataHub deployment changes\n\nSwitch to using Google Cloud SQL, which provides hosted and managed MySQL database\nPerform regular and tested backups of the database\nStart writing an operational FAQ for things to do and not do\nSetup better monitoring and paging systems\nDocument escalation procedures explicitly"
   },
   {
-    "objectID": "incidents/2018-02-06-hub-db-dir.html",
-    "href": "incidents/2018-02-06-hub-db-dir.html",
-    "title": "Azure PD refuses to detach, causing downtime for data100",
+    "objectID": "incidents/2024-core-node-incidents.html",
+    "href": "incidents/2024-core-node-incidents.html",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
     "section": "",
-    "text": "On February 5, 2018, a PR was merged into the production cluster for Data 100. The CI got as far as running helm upgrade but the hub’s persistent volume would not detach from the old hub. The new hub pod had to wait on the hub-db-dir volume and so would not start. The persistent volume claim was ultimately deleted. The subsequent helm upgrade created a new volume and a new hub pod was able to start."
+    "text": "Over the past couple of years, all of our production hubs have been having persistent issues with our core nodes having major load spikes during ‘peak’ usage and the impacted node (which hosts all of our hub and proxy pods – not user pods) crashing. This would then impact every hub, causing all users to see 503 http errors until a new node finishing spinning up. We also suspect that the ‘white screen’ issue some users see after logging in is related to this.\nThese outages would usually last anywhere from 45 to 90+ minutes. The first chunk of time would be the core node getting wedged and eventually dying, and the last 15-20 minutes would be spent on the new node spinning up and services restarting.\nMany of these incidents are tracked here.\nWe have spent much time working to debug and track this, including with our friends at 2i2c. After much deep-diving and debugging, we were able to narrow this down to a memory (socket?) leak in the configurable http proxy.\nAfter some back and forth w/the upstream maintainers, we received a forked version of the proxy to test.\nDuring this testing, we triggered some user-facing downtime, as well as the proxy itself crashing and causing small outages.\nAnother (unrelated) issue that impacted users was that GKE was autoscaling our core pool (where the hub and proxy pods run) node to zero. Since it takes about 10-15m for a new node to spin up, all hubs were inaccessible until the new node was deployed.\nUser Impact:\n\n\nOn the afternoon of Feb 7th, I was testing the fork of the proxy + some revised timeouts on the Data 8 hub. This caused the proxy to crash every ~20m over the course of a few hours. I then reverted the fork and timeout changes.\nDuring the latter half of February, our core pool was being autoscaled from 1 to 0 nodes. This caused multiple short outages.\nThe proxy pods for Data 8 and Data 100 (our largest classes) crashed continually under load (~250+ simultaneous users), causing users to receive 500 HTTP errors until the pod automatically restarted."
   },
   {
-    "objectID": "incidents/2018-02-06-hub-db-dir.html#summary",
-    "href": "incidents/2018-02-06-hub-db-dir.html#summary",
-    "title": "Azure PD refuses to detach, causing downtime for data100",
+    "objectID": "incidents/2024-core-node-incidents.html#summary",
+    "href": "incidents/2024-core-node-incidents.html#summary",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
     "section": "",
-    "text": "On February 5, 2018, a PR was merged into the production cluster for Data 100. The CI got as far as running helm upgrade but the hub’s persistent volume would not detach from the old hub. The new hub pod had to wait on the hub-db-dir volume and so would not start. The persistent volume claim was ultimately deleted. The subsequent helm upgrade created a new volume and a new hub pod was able to start."
+    "text": "Over the past couple of years, all of our production hubs have been having persistent issues with our core nodes having major load spikes during ‘peak’ usage and the impacted node (which hosts all of our hub and proxy pods – not user pods) crashing. This would then impact every hub, causing all users to see 503 http errors until a new node finishing spinning up. We also suspect that the ‘white screen’ issue some users see after logging in is related to this.\nThese outages would usually last anywhere from 45 to 90+ minutes. The first chunk of time would be the core node getting wedged and eventually dying, and the last 15-20 minutes would be spent on the new node spinning up and services restarting.\nMany of these incidents are tracked here.\nWe have spent much time working to debug and track this, including with our friends at 2i2c. After much deep-diving and debugging, we were able to narrow this down to a memory (socket?) leak in the configurable http proxy.\nAfter some back and forth w/the upstream maintainers, we received a forked version of the proxy to test.\nDuring this testing, we triggered some user-facing downtime, as well as the proxy itself crashing and causing small outages.\nAnother (unrelated) issue that impacted users was that GKE was autoscaling our core pool (where the hub and proxy pods run) node to zero. Since it takes about 10-15m for a new node to spin up, all hubs were inaccessible until the new node was deployed.\nUser Impact:\n\n\nOn the afternoon of Feb 7th, I was testing the fork of the proxy + some revised timeouts on the Data 8 hub. This caused the proxy to crash every ~20m over the course of a few hours. I then reverted the fork and timeout changes.\nDuring the latter half of February, our core pool was being autoscaled from 1 to 0 nodes. This caused multiple short outages.\nThe proxy pods for Data 8 and Data 100 (our largest classes) crashed continually under load (~250+ simultaneous users), causing users to receive 500 HTTP errors until the pod automatically restarted."
   },
   {
-    "objectID": "incidents/2018-02-06-hub-db-dir.html#timeline",
-    "href": "incidents/2018-02-06-hub-db-dir.html#timeline",
-    "title": "Azure PD refuses to detach, causing downtime for data100",
+    "objectID": "incidents/2024-core-node-incidents.html#hub-information",
+    "href": "incidents/2024-core-node-incidents.html#hub-information",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
+    "section": "Hub information",
+    "text": "Hub information\n\nData 8\nData 100"
+  },
+  {
+    "objectID": "incidents/2024-core-node-incidents.html#timeline",
+    "href": "incidents/2024-core-node-incidents.html#timeline",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
     "section": "Timeline",
-    "text": "Timeline\n\n2018-02-05 20:57\nA PR for data100 is merged.\n\n\n21:20\nTowards the end of the build, the upgrade fails because the new hub pod does not start up. The hub-db-dir volume remains bound to the old hub pod which is stuck in a Terminating state. The hub pod only completes termination when delete is passed a grace period of 0. The hub volume remains bound however.\n\n\n21:30\nCI is restarted but by the time helm is run, the hub-db-dir volume remains bound and cannot be attached to the new hub pod. Additionally, helm errors because the jupyterhub-internal ingress object cannot be found even though it does exist.\n\n\n21:45\nSince it cannot be determined what node the volume is bound to, the volume is deleted. The jupyterhub-internal ingress object is also deleted prior to restarting the CI build.\n\n\n22:05\nThe hub comes up with a new hub-db-dir volume. CI fails due to the same jupyterhub-internal object error."
+    "text": "Timeline\n\n2024-03-05 Data8 outage\nbetween ~4pm and ~5:15pm, data8’s configurable-http-proxy (chp) was oomkilled and caused many 503 errors to be issued to the users. here’s a rough timeline of what happened (culled from grafana, gcp logs and kernel logs on the core node):\n\n\n15:12:00\n~280 concurrent users\n\n\n15:12:42\nchp “uncaught exception: write EPIPE”\n\n\n16:00:00\nproxy ram 800Mi (steady)\n\n\n16:05:00\n~300 concurrent users\n\n\n~16:05:00\nspike on proxy — cpu 181%, mem 1.06Gi –&gt; 1.86Gi\n\n\n16:05:53\nchp healthz readiness probe failure\n\n\n16:05:56\nchp/javascript runs out of heap “Ineffective mark-compass near heap limit Allocation Failed”\n\n\n16:05:57-58\nchp restarts\n\n\n16:05:57\nnode dmesg: “TCP: request_sock_TCP: Possible SYN flooding on port 8000. Sending cookies. Check SNMP counters.”\n\n\n~16:06:00\n2.5K 503 errors\n\n\n~16:06:00 - 16:16:00\nmany many chp routes added and deleted causing 503 errors for some users\n\n\n16:16:00 - 16:49:00\neverything back to normal\n\n\n16:49:00\n~300 users (slowly decreasing)\n\n\n~16:49:00\nspike on proxy — cpu 107%, mem 814Mi –&gt; 2.45Gi\n\n\n16:49:40\ncore node dmesg: node invoked oom-killer: gfp_mask=0xcc0(GFP_KERNEL), order=0, oom_score_adj=999\n\n\n16:50:25\nchp restarts (no heap error)\n\n\n~16:50:00\n5.7K 503 errors\n\n\n16:54:15 - 17:15:31\n300 users (slowly decreasing), 3x chp “uncaught exception: write EPIPE”, intermittent 503 errors in spikes of 30, 60, 150, hub latency 2.5sec\n\n\n18:47:19 - 18:58:10\n~120 users (constant), 3x chp “uncaught exception: write EPIPE”, intermittent 503 errors in spikes of 30, 60, hub latency 3sec\n\n\n~19:00\nthings return to normal"
   },
   {
-    "objectID": "incidents/2018-02-06-hub-db-dir.html#conclusion",
-    "href": "incidents/2018-02-06-hub-db-dir.html#conclusion",
-    "title": "Azure PD refuses to detach, causing downtime for data100",
-    "section": "Conclusion",
-    "text": "Conclusion\nAzure was not able to detach the hub-db-dir azure disk from the hub pod. The pvc is deleted and the hub comes up on the next CI run."
+    "objectID": "incidents/2024-core-node-incidents.html#what-went-wrong",
+    "href": "incidents/2024-core-node-incidents.html#what-went-wrong",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
+    "section": "What went wrong",
+    "text": "What went wrong\n\nOnly being able to test things like the forked chp in prod is dangerous and has a huge potential impact on users.\nThe configurable http proxy (chp) is written in javascript (nodejs), and under load (~250+ concurrent users) it leaks ports and eventually fills up the heap (~728M) and gets oomkilled.\nThe CPU allocation on the core node was also spiking, potentially causing login latency across all hubs.\nUpon chp restart, it can take up to 10-15m for the routing table to be repopulated. During this time most, if not all, users that were already connected to the hub will get 503 errors. Any new logins during this time will not."
   },
   {
-    "objectID": "incidents/2018-02-06-hub-db-dir.html#action-items",
-    "href": "incidents/2018-02-06-hub-db-dir.html#action-items",
-    "title": "Azure PD refuses to detach, causing downtime for data100",
+    "objectID": "incidents/2024-core-node-incidents.html#where-we-got-lucky",
+    "href": "incidents/2024-core-node-incidents.html#where-we-got-lucky",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
+    "section": "Where we got lucky",
+    "text": "Where we got lucky\n\nSince we bumped the RAM allocated to the core node from 8G to 32G instances like are isolated to whatever hub’s chp begins failing, and does not impact all hubs."
+  },
+  {
+    "objectID": "incidents/2024-core-node-incidents.html#action-items",
+    "href": "incidents/2024-core-node-incidents.html#action-items",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nStore the hub db in a cloud database to eliminate reliance on hub volume.\nDowngrade helm to 2.6.x to see if this fixes the helm upgrades."
+    "text": "Action items\n\nProcess/Policy improvements\n\nWork with instructors to identify and immediately troubleshoot directly with users impacted by the white screen issue.\nSpin up a dev hub and figure out how to use hubtraf to simulate a large number of users doing work.\n\n\n\nDocumentation improvements\nNone.\n\n\nTechnical improvements\n\nContinue to track the port leak issue upstream.\nDeploy a new core pool with the same RAM and more CPU (Jira DH-259).\nSpin up a dev hub and figure out how to use hubtraf to simulate a large number of users doing work."
   },
   {
-    "objectID": "incidents/2017-05-09-gce-billing.html",
-    "href": "incidents/2017-05-09-gce-billing.html",
-    "title": "Oops we forgot to pay the bill",
+    "objectID": "incidents/2024-core-node-incidents.html#actions",
+    "href": "incidents/2024-core-node-incidents.html#actions",
+    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
+    "section": "Actions",
+    "text": "Actions\n\nIncident has been dealt with or is over\nSections above are filled out\nIncident title and after-action report is cleaned up\nAll actionable items above have linked GitHub Issues"
+  },
+  {
+    "objectID": "incidents/2018-06-11-course-subscription-canceled.html",
+    "href": "incidents/2018-06-11-course-subscription-canceled.html",
+    "title": "Azure billing issue causes downtime",
     "section": "",
-    "text": "On May 9, 2017, the compute resources associated with the data-8 project at GCE were suspended. All hubs including datahub, stat28, and prob140 were not reachable. This happened because the grant that backed the project’s billing account ran out of funds. The project was moved to a different funding source and the resources gradually came back online."
+    "text": "On June 11, 2018, the cloud vendor notified ds-instr that the data8-17f-prod subscription was canceled due to its usage cap. The educator portal confirmed that the spend had surpassed the budget. After additional funds were allocated to the subscription, a portion of the VMs were manually started. The hub came back online after pods were forcibly deleted and nodes were cordoned."
   },
   {
-    "objectID": "incidents/2017-05-09-gce-billing.html#summary",
-    "href": "incidents/2017-05-09-gce-billing.html#summary",
-    "title": "Oops we forgot to pay the bill",
+    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#summary",
+    "href": "incidents/2018-06-11-course-subscription-canceled.html#summary",
+    "title": "Azure billing issue causes downtime",
     "section": "",
-    "text": "On May 9, 2017, the compute resources associated with the data-8 project at GCE were suspended. All hubs including datahub, stat28, and prob140 were not reachable. This happened because the grant that backed the project’s billing account ran out of funds. The project was moved to a different funding source and the resources gradually came back online."
+    "text": "On June 11, 2018, the cloud vendor notified ds-instr that the data8-17f-prod subscription was canceled due to its usage cap. The educator portal confirmed that the spend had surpassed the budget. After additional funds were allocated to the subscription, a portion of the VMs were manually started. The hub came back online after pods were forcibly deleted and nodes were cordoned."
   },
   {
-    "objectID": "incidents/2017-05-09-gce-billing.html#timeline",
-    "href": "incidents/2017-05-09-gce-billing.html#timeline",
-    "title": "Oops we forgot to pay the bill",
+    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#timeline",
+    "href": "incidents/2018-06-11-course-subscription-canceled.html#timeline",
+    "title": "Azure billing issue causes downtime",
     "section": "Timeline",
-    "text": "Timeline\n\n2017-05-09 16:51\nA report in the Data 8 Spring 2017 Staff slack, #jupyter channel, says that datahub is down. This is confirmed. Attempting to access the provisioner via gcloud compute ssh provisioner-01 fails with:\nERROR: (gcloud.compute.ssh) Instance [provisioner-01] in zone [us-central1-a] has not been allocated an external IP address yet. Try rerunning this command later.\n\n\n17:01\nThe Google Cloud console shows that the billing account has run out of the grant that supported the data-8 project. The project account is moved to another billing account which has resources left.\nThe billing state is confirmed by gcloud messages:\nGoogle Compute Engine: Project data-8 cannot accept requests to setMetadata while in an inactive billing state.  Billing state may take several minutes to update.\n\n\n17:09\nprovisioner-01 is manually started. All pods in the datahub namespace are deleted.\n\n\n17:15\ndatahub is back online. stat28 and prob140 hub pods are manually killed. After a few moments the hubs are back online. The autoscaler is started.\n\n\n17:19\nThe slack duplicator is started.\n\n\n2017-05-10 10:48\nA report in uc-jupyter #jupyterhub says that try.datahub is down. This is confirmed and the hub in the tmp namespace is killed. The hub comes online a couple of minutes later."
+    "text": "Timeline\n\n2018-06-11 9:02a\nds-instr received an email from the cloud vendor:\n\nThe following subscriptions under your Microsoft Azure sponsorships for ds-instr@berkeley.edu have recently become canceled. Because these subscription(s) are canceled, all services have been suspended but no data has been lost. You have 90 days from the date of cancellation before Microsoft will delete the subscription and all attached data. Please use the Educator Portal to reactivate the subscription(s).\n\n\n\n\nSubscription Name\nSubscription Id\nCanceled Reason\n\n\n\n\ndata8-17f-prod\nomitted here\nSubscription Cap\n\n\n\n\n\n9:29\nThe subscription status was confirmed at https://www.microsoftazuresponsorships.com/Manage. In order to allocate additional budget to data8-17f-prod, budget for other subscriptions had to be reduced.\n\n\n9:40\nVMs were turned on at https://portal.azure.com: 3 nodes in each node pool, the nfs server, the kubernetes master, and the database server.\n\n\n9:45\nThe hub was unreachable even though the VMs were online. The hub and proxy pods were shown as Running and all nodes were shown as online even though some nodes had not been started. The offline cluster nodes were manually cordoned. All pods had to be forcibly deleted before they would start.\n\n\n10:14\nThe Billing Alert Service was checked at https://account.azure.com/Subscriptions/alert?subscriptionId=06f94ac5-b029-411f-8896-411f3c6778b4 and it was discovered that alerts were no longer registered."
   },
   {
-    "objectID": "incidents/2017-05-09-gce-billing.html#conclusion",
-    "href": "incidents/2017-05-09-gce-billing.html#conclusion",
-    "title": "Oops we forgot to pay the bill",
+    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#conclusion",
+    "href": "incidents/2018-06-11-course-subscription-canceled.html#conclusion",
+    "title": "Azure billing issue causes downtime",
     "section": "Conclusion",
-    "text": "Conclusion\nThere was insufficient monitoring of the billing status."
+    "text": "Conclusion\nThere were insufficient funds on the subscription to persist its resources. The subscription budget was increased and the hub was brought back online. The billing alert service that was configured to prevent such incidents did not function properly."
   },
   {
-    "objectID": "incidents/2017-05-09-gce-billing.html#action-items",
-    "href": "incidents/2017-05-09-gce-billing.html#action-items",
-    "title": "Oops we forgot to pay the bill",
+    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#action-items",
+    "href": "incidents/2018-06-11-course-subscription-canceled.html#action-items",
+    "title": "Azure billing issue causes downtime",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nIdentify channels for billing alerts.\nIdentify billing threshold functions that predict when funds will run out.\nEstablish off-cloud backups. The plan is to do this via nbgdrive.\nStart autoscaler automatically. It is manually started at the moment.\n\n\n\nMonitoring\n\nSetup scheduled billing reports and threshold alarms.\nSetup hub monitoring!\nThe slack duplicator runs in one of the GCP clusters. When the clusters go down, slack messages aren’t forwarded from the data8-sp17-staff slack to uc-jupyter."
-  },
-  {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html",
-    "title": "Hubs throwing 505 errors",
-    "section": "",
-    "text": "PR 1 and PR 2 were merged to prod between 2 AM and 2.30 AM PST on 1/20. Difference due to the commits can be viewed here\nDue to these changes, image rebuild happened which broke multiple hubs which used that image including Datahub, ISchool, R, Data 100 and Data 140 hubs.\nOne of the dependencies highlighted as part of the image build had an upgrade which resulted in R hub throwing 505 error and Data 100/140 hub throwing “Error starting Kernel”. [Yuvi to fill in the right technical information]\nUser Impact:\n\n\nR Hub was not accessible for about 6 hours. Issue affected 10+ Stat 20 GSIs planning for their first class of the semester (catering to the needs of 600+ students). Hub went down for few minutes during the instruction.\nProb 140 hub was not available till 12.15 AM PST\nData 100 hub was not available till 12.33 AM. Thankfully, assignments were not due till friday (1/21)\nFew users in Ischool were affected as they could not access R Studio"
-  },
-  {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#summary",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#summary",
-    "title": "Hubs throwing 505 errors",
-    "section": "",
-    "text": "PR 1 and PR 2 were merged to prod between 2 AM and 2.30 AM PST on 1/20. Difference due to the commits can be viewed here\nDue to these changes, image rebuild happened which broke multiple hubs which used that image including Datahub, ISchool, R, Data 100 and Data 140 hubs.\nOne of the dependencies highlighted as part of the image build had an upgrade which resulted in R hub throwing 505 error and Data 100/140 hub throwing “Error starting Kernel”. [Yuvi to fill in the right technical information]\nUser Impact:\n\n\nR Hub was not accessible for about 6 hours. Issue affected 10+ Stat 20 GSIs planning for their first class of the semester (catering to the needs of 600+ students). Hub went down for few minutes during the instruction.\nProb 140 hub was not available till 12.15 AM PST\nData 100 hub was not available till 12.33 AM. Thankfully, assignments were not due till friday (1/21)\nFew users in Ischool were affected as they could not access R Studio"
+    "text": "Action items\n\nProcess\n\nDo not use subscription portal billing alerts.\nManually check subscription usage via an unattended process."
   },
   {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#hub-information",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#hub-information",
-    "title": "Hubs throwing 505 errors",
-    "section": "Hub information",
-    "text": "Hub information\n\nHub URL: https://r.datahub.berkeley.edu/ and most other hubs highlighted above"
+    "objectID": "incidents/2017-10-19-course-subscription-canceled.html",
+    "href": "incidents/2017-10-19-course-subscription-canceled.html",
+    "title": "Billing confusion with Azure portal causes summer hub to be lost",
+    "section": "",
+    "text": "On October 10, 2017, the cloud vendor notified ds-instr that the data8r-17s subscription was canceled due to its end date, and we had 90 days to reactivate it using the educator portal. A support ticket was created to reverse the cancellation since the educator portal did not permit reactivation. On October 18 we were notified that the subscription’s resources were deleted.\nCoincidentally, a script was written on Oct. 9 to backup data to ds-instr’s Google Drive and it was performed for the instructor as a test. Unfortunately it wasn’t run for all users before the resources were taken offline."
   },
   {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#timeline-if-relevant",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#timeline-if-relevant",
-    "title": "Hubs throwing 505 errors",
-    "section": "Timeline (if relevant)",
-    "text": "Timeline (if relevant)\n\n{{ 2022-01-20 Between 02:00 and 02.30 PM }}\nPR 1 and PR 2 were merged to prod. Notably, PR 1 had multiple commits related to creation of Stat 20 hub, Stat 259 hub etc..\n\n\n{{ 06:10 }}\nAndrew Bray (Stat 20 instructor) raised a github issue around 5.45 AM PST.\n\n\n{{ 07:45 }}\nYuvi quickly jumped in to make a fix to get the R hub working. However this fix resulted in breaking Stat 20 hub.\n\n\n{{ 07:53 }}\nISchool folks reported issues with using RStudio in Datahub\n\n\n{{ 08:45 }}\nYuvi fixed issue with Stat 20 and other hubs\n\n\n{{ 12:10 }}\nGSIs from Data 100 and 140 reported “Unhandled error” in their hubs\n\n\n{{ 12:15 }}\nGSIs for Data 140 hub reported that the error was fixed\n\n\n{{ 12:33 }}\nGSIs Data 100 hub reported that the error was fixed"
+    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#summary",
+    "href": "incidents/2017-10-19-course-subscription-canceled.html#summary",
+    "title": "Billing confusion with Azure portal causes summer hub to be lost",
+    "section": "",
+    "text": "On October 10, 2017, the cloud vendor notified ds-instr that the data8r-17s subscription was canceled due to its end date, and we had 90 days to reactivate it using the educator portal. A support ticket was created to reverse the cancellation since the educator portal did not permit reactivation. On October 18 we were notified that the subscription’s resources were deleted.\nCoincidentally, a script was written on Oct. 9 to backup data to ds-instr’s Google Drive and it was performed for the instructor as a test. Unfortunately it wasn’t run for all users before the resources were taken offline."
   },
   {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#what-went-wrong",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#what-went-wrong",
-    "title": "Hubs throwing 505 errors",
-    "section": "What went wrong",
-    "text": "What went wrong\n\nR, Stat 20, Datahub, ISchool, Data 100 and 140 hubs went down around 2.30 AM PST. However, the team was aware of these issues only when users reported errors at different time intervals (as listed above)\nMultiple commits went through a single PR. Dependency package’s version upgrade broke the image build (Yuvi to fill in the required details)\n\nThings that could have gone better. Ideally these should result in concrete action items that have GitHub issues created for them and linked to under Action items."
+    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#timeline",
+    "href": "incidents/2017-10-19-course-subscription-canceled.html#timeline",
+    "title": "Billing confusion with Azure portal causes summer hub to be lost",
+    "section": "Timeline",
+    "text": "Timeline\n\n2017-10-10 9:06a\nds-instr received an email from the cloud vendor:\n\nThe following subscriptions under your [cloud vendor] sponsorships for ds-instr@berkeley.edu have recently become canceled. Because these subscription(s) are canceled, all services have been suspended but no data has been lost. You have 90 days from the date of cancellation before [the cloud vendor] will delete the subscription and all attached data. Please use the Educator Portal to reactivate the subscription(s).\n\n\n\n\nSubscription Name\nSubscription Id\nCanceled Reason\n\n\n\n\ndata8r-17s\nomitted here\nSubscription End Date\n\n\n\n\n\n9:30\nThe instructor was notified. The educator portal did not provide a way to view or alter the subscription end date of a canceled subscription so a support request was filed at the cloud vendor.\n\n\n11:14\nThe cloud vendor asks that a payment instrument be added to the ds-instr account. We respond that the account is funded by a sponsorship.\n\n\n17:22\nThe cloud vendor contacts their sponsorship team.\n\n\n2017-10-11 15:00\nThe cloud vendor calls to discuss the situation. Screenshots of the educator and cloud portal were sent to the cloud vendor.\n\n\n2017-10-12 16:19\nThe cloud vendor offers to enable the subscription for a 60 minute period from the backend so that the End Date may be extended from the portal. Though the subscription is re-enabled for an hour, the portal still does not permit the subscription parameters to be changed.\n\n\n2017-10-18 15:29\nThe cloud vendor says that the subscription was actually disabled because it had exhausted the allocated funds, and the data was deleted within 24 hours despite the stated 90 day grace period. Later the following was provided by the cloud vendor:\n\nI worked with our backend engineering team last night and I am afraid to say that we could not retrieve the storage account after all our sincere efforts. I understand how frustrating it would be for you and I do not have the words to express the same, I just wish if I could be of some help to you.\nHaving said that we did dig into the reasons behind this situation, the subscription was initially suspended by an internal engineering job occurred that auto-suspended all Academic Account Sponsorship subscriptions with an end date that was part of the previous fiscal year. Usually this suspension does not delete the subscription. There are a few [cloud vendor] accounts which are on legacy commerce platform which are affected and these accounts are in the process of modern platform. Your account was in the transition mode when the subscription got suspended and your account was partially converted to the modern platform. The billing & subscription part was converted to the modern platform but the not at the service level. Hence you got the message that your data would be retained for 90 days, at the same stated at the service level it was not converted to the modern hence the data got deleted.\nI had a detailed discussion with our product group team on this and how we can avoid this in future. First of all, your account is now completely migrated/transitioned completely to the modern platform. Also, to ensure that our other Academic Account Sponsorships customers do not face the same issue they have agreed to complete the migration manually on those accounts.\n\n\n\n2017-10-19 10:53\nThe cloud vendor compensates ds-instr with an additional $10k for the experience."
   },
   {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#where-we-got-lucky",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#where-we-got-lucky",
-    "title": "Hubs throwing 505 errors",
-    "section": "Where we got lucky",
-    "text": "Where we got lucky\nThese are good things that happened to us but not because we had planned for them.\n\nYuvi was awake at the time when issue was reported and was able to fix the issues immediately.\nClasses using hubs were not completely affected due to this outage (Data 100 did not have assignments due till 1/21 and Stat 20 had few mins of outage during instruction)"
+    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#conclusion",
+    "href": "incidents/2017-10-19-course-subscription-canceled.html#conclusion",
+    "title": "Billing confusion with Azure portal causes summer hub to be lost",
+    "section": "Conclusion",
+    "text": "Conclusion\nThere were insufficient funds on the subscription to persist its resources. The resources were deleted by the cloud vendor before the grace period ran out."
   },
   {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#action-items",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#action-items",
-    "title": "Hubs throwing 505 errors",
+    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#action-items",
+    "href": "incidents/2017-10-19-course-subscription-canceled.html#action-items",
+    "title": "Billing confusion with Azure portal causes summer hub to be lost",
     "section": "Action items",
-    "text": "Action items\nThese are only sample subheadings. Every action item should have a GitHub issue (even a small skeleton of one) attached to it, so these do not get forgotten. These issues don’t have to be in infrastructure/, they can be in other repositories.\n\nProcess/Policy improvements\n\n{{Develop manual testing process whenever a PR gets merged to staging of the major hubs (till automated test suites are written)}} link to github issue]\nDevelop a policy around when to create a new hub and what type of changes get deployed to Datahub!\n\n\n\nDocumentation improvements\n\n{{ Start writing after action reports for future outages }} [link to github issue]\n{{ summary }} [link to github issue]\n\n\n\nTechnical improvements\n\n{{ Enabling logging mechanism across all hubs to track future outages }}\n{{ Adapt 2I2C testing suite to develop automated test cases that check the sanity of the different services whenever a PR gets merged in staging}} [link to github issue]\n{{ Investigate the reason why pager duty did not throw an alert for 5xx errors when the hubs went down. Fix the alerting mechanism so that they notify all kind of errors }} [link to github issue]\n{{ Adding R Studio as part of Repo2Docker}} [link to github issue]"
+    "text": "Action items\n\nProcess\n\nUntil there is a per-user backup implemented hub-side, set a schedule for backing up user data for every course.\nAlways set a billing alert at some conservative amount less than the subscription alotment\nIf a subscription is ever canceled, backup user data within 24 hours, regardless of the stated grace period."
   },
   {
-    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#actions",
-    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#actions",
-    "title": "Hubs throwing 505 errors",
-    "section": "Actions",
-    "text": "Actions\n\nIncident has been dealt with or is over\nSections above are filled out\nIncident title and after-action report is cleaned up\nAll actionable items above have linked GitHub Issues"
+    "objectID": "incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html",
+    "href": "incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html",
+    "title": "",
+    "section": "",
+    "text": "import pandas as pd\nimport numpy as np\n\n\n!head pvcs.csv\n\nNAME,STATUS,VOLUME,CAPACITY,ACCESSMODES,AGE\nclaim-redacted-205,Bound,pvc-5643aadf-ddb3-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-1088,Bound,pvc-bc68d752-ecba-11e6-98ef-42010af000c3,10Gi,RWO,2d\nclaim-redacted-389,Bound,pvc-5d67d43e-ddcb-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-119,Bound,pvc-73c5d644-ddb1-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-64,Bound,pvc-3c2345e4-dd9f-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-192,Bound,pvc-51b672a8-ddb3-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-814,Bound,pvc-2b83ae02-dee0-11e6-98ef-42010af000c3,10Gi,RWO,20d\nclaim-redacted-775,Bound,pvc-ec1d47be-deb0-11e6-98ef-42010af000c3,10Gi,RWO,20d\nclaim-redacted-954,Bound,pvc-5a0acdc5-e3ee-11e6-98ef-42010af000c3,10Gi,RWO,13d\n\n\n\ndf = pd.DataFrame.from_csv('pvcs.csv')\ndf.head()\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\n\n\nNAME\n\n\n\n\n\n\n\n\n\nclaim-redacted-205\nBound\npvc-5643aadf-ddb3-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\nclaim-redacted-1088\nBound\npvc-bc68d752-ecba-11e6-98ef-42010af000c3\n10Gi\nRWO\n2d\n\n\nclaim-redacted-389\nBound\npvc-5d67d43e-ddcb-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\nclaim-redacted-119\nBound\npvc-73c5d644-ddb1-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\nclaim-redacted-64\nBound\npvc-3c2345e4-dd9f-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\n\n\n\n\n\n\ndf['username'] = np.array(df.index.str.extract('\\w+-([\\w-]+)-\\d+$'))\ndf['id'] = np.array(df.index.str.extract('\\w+-\\w+-(\\d+)$'))\ndf.head()\n\n/Users/redacted/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:1: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n  if __name__ == '__main__':\n/Users/redacted/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:2: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n  from ipykernel import kernelapp as app\n\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\nusername\nid\n\n\n\nNAME\n\n\n\n\n\n\n\n\n\n\n\n\nclaim-redacted-205\nBound\npvc-5643aadf-ddb3-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n\n205\n\n\nclaim-redacted-1088\nBound\npvc-bc68d752-ecba-11e6-98ef-42010af000c3\n10Gi\nRWO\n2d\nredacted\n1088\n\n\n\nclaim-redacted-389\nBound\npvc-5d67d43e-ddcb-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n389\n\n\n\nclaim-redacted-119\nBound\npvc-73c5d644-ddb1-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n119\n\n\n\nclaim-redacted-64\nBound\npvc-3c2345e4-dd9f-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n64\n\n\n\n\n\n\n\n\n\ndf.tail()\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\nusername\nid\n\n\nNAME\n\n\n\n\n\n\n\n\n\n\n\nclaim-redacted-562\nBound\npvc-4418d1d6-de0a-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n562\n\n\nclaim-redacted-400\nBound\npvc-2399fe54-ddcf-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n400\n\n\nclaim-redacted-946\nBound\npvc-294cb2c7-e35e-11e6-98ef-42010af000c3\n10Gi\nRWO\n14d\nredacted\n946\n\n\nclaim-redacted-900\nBound\npvc-f6859607-e1c0-11e6-98ef-42010af000c3\n10Gi\nRWO\n16d\nredacted\n900\n\n\nhub-db-dir\nBound\npvc-ea8a0bef-dd44-11e6-98ef-42010af000c3\n10Gi\nRWO\n22d\nNaN\nNaN\n\n\n\n\n\n\n\n\nvalids = df[~df['AGE'].str.contains('m')].dropna()\nvalids = valids[valids['id'] != '14']\nvalids.head()\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\nusername\nid\n\n\nNAME\n\n\n\n\n\n\n\n\n\n\n\nclaim-redacted-205\nBound\npvc-5643aadf-ddb3-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n205\n\n\nclaim-redacted-1088\nBound\npvc-bc68d752-ecba-11e6-98ef-42010af000c3\n10Gi\nRWO\n2d\nredacted\n1088\n\n\nclaim-redacted-389\nBound\npvc-5d67d43e-ddcb-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n389\n\n\nclaim-redacted-119\nBound\npvc-73c5d644-ddb1-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n119\n\n\nclaim-redacted-64\nBound\npvc-3c2345e4-dd9f-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n64\n\n\n\n\n\n\n\n\nvalids['username'].head().value_counts()\n\nredacted              1\nredacted    1\nredacted              1\nredacted            1\nredacted         1\nName: username, dtype: int64\n\n\n\nvalids['id'].value_counts().head()\n\n400     1\n90      1\n756     1\n488     1\n1029    1\nName: id, dtype: int64\n\n\n\nlen(valids['id'])\n\n1105\n\n\n\nimport sqlite3\nconn = sqlite3.connect('jupyterhub.sqlite')\nc = conn.cursor()\n\n\nc.execute('PRAGMA TABLE_INFO({})'.format('users'))\n\nc.fetchall()\n\n[(0, 'id', 'INTEGER', 1, None, 1),\n (1, 'name', 'VARCHAR(1023)', 0, None, 0),\n (2, '_server_id', 'INTEGER', 0, None, 0),\n (3, 'admin', 'BOOLEAN', 0, None, 0),\n (4, 'last_activity', 'DATETIME', 0, None, 0),\n (5, 'cookie_id', 'VARCHAR(1023)', 0, None, 0),\n (6, 'state', 'TEXT', 0, None, 0),\n (7, 'auth_state', 'TEXT', 0, None, 0)]\n\n\n\nc.execute('SELECT * FROM users LIMIT 20')\ncurrent_users = c.fetchall()\ncurrent_users[:2]\n\n[(1,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  '8ec09f8def774b668e838d1e70e00329',\n  None,\n  None),\n (2,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.938610',\n  'c98f59afc1fe4a70b18b451ac1190d45',\n  None,\n  None)]\n\n\n\nadmins = {user[1] for user in current_users if user[3] == 1}\nadmins\n\n{'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted'}\n\n\n\n# I just did this manually in the sqlite3 CLI\n# c.execute('DROP FROM users where id &gt; -1')\n# c.fetchall()\n\n\n---------------------------------------------------------------------------\nOperationalError                          Traceback (most recent call last)\n&lt;ipython-input-73-a8eb9e4f2c5d&gt; in &lt;module&gt;()\n----&gt; 1 c.execute('DROP FROM users where id &gt; -1')\n      2 c.fetchall()\n\nOperationalError: near \"FROM\": syntax error\n\n\n\n\nimport datetime\nimport itertools\n\nrecords = list(zip(\n    valids['id'].astype(int),\n    valids['username'],\n    itertools.repeat('NULL'),\n    [1 if name in admins else 0 for name in valids['username']],\n#     itertools.repeat('NULL'), # Pick a random valid value\n    itertools.repeat('2017-02-09 09:07:03.936620'), # Pick a random valid value\n    itertools.repeat('thisisadummycookiehopefullyitworks'),\n    itertools.repeat('NULL'),\n    itertools.repeat('NULL')\n))\nrecords[:3]\n\n[(205,\n  'redacted',\n  'NULL',\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  'NULL',\n  'NULL'),\n (1088,\n  'redacted',\n  'NULL',\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  'NULL',\n  'NULL'),\n (389,\n  'redacted',\n  'NULL',\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  'NULL',\n  'NULL')]\n\n\n\n# I hate everything\ndef record_to_sql(record):\n    return \"INSERT INTO users VALUES({}, '{}', {}, '{}', '{}', '{}', {}, {})\".format(\n        *record\n    )\n\nsql_statements = [record_to_sql(record) for record in records]\nrecord_to_sql(records[0])\n\n\"INSERT INTO users VALUES(205, 'redacted', NULL, '0', '2017-02-09 09:07:03.936620', 'thisisadummycookiehopefullyitworks', NULL, NULL)\"\n\n\n[(0, 'id', 'INTEGER', 1, None, 1),\n (1, 'name', 'VARCHAR(1023)', 0, None, 0),\n (2, '_server_id', 'INTEGER', 0, None, 0),\n (3, 'admin', 'BOOLEAN', 0, None, 0),\n (4, 'last_activity', 'DATETIME', 0, None, 0),\n (5, 'cookie_id', 'VARCHAR(1023)', 0, None, 0),\n (6, 'state', 'TEXT', 0, None, 0),\n (7, 'auth_state', 'TEXT', 0, None, 0)]\n\nwith conn:\n    c = conn.cursor()\n    for statement in sql_statements:\n        c.execute(statement)\n\n\nc.execute('SELECT * FROM users LIMIT 20')\nc.fetchall()\n\n[(2,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (3,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (4,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (5,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (6,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (7,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (8,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (9,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (10,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (11,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (12,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (13,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (15,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (17,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (18,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (19,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (20,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (21,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (22,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (24,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None)]"
   },
   {
     "objectID": "incidents/2019-02-25-k8s-api-server-down.html",
@@ -1194,388 +1183,351 @@
     "text": "Update\nCloud provider’s response on 3/15/2019:\n\nAfter reviewing all the logs we have, our backend advised below.\n\nWe’ve identified that there were problems with the infrastructure hosting your cluster which caused the kubelet on the master stopped responding. There were alerts regarding this issue which were addressed by our teams. We’re working to reduce the impact of these events as much as possible.\n\nPlease be advised this is not related with region stability\nFeel free to let me know if any further questions and thanks for your patience."
   },
   {
-    "objectID": "incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html",
-    "href": "incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html",
-    "title": "",
-    "section": "",
-    "text": "import pandas as pd\nimport numpy as np\n\n\n!head pvcs.csv\n\nNAME,STATUS,VOLUME,CAPACITY,ACCESSMODES,AGE\nclaim-redacted-205,Bound,pvc-5643aadf-ddb3-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-1088,Bound,pvc-bc68d752-ecba-11e6-98ef-42010af000c3,10Gi,RWO,2d\nclaim-redacted-389,Bound,pvc-5d67d43e-ddcb-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-119,Bound,pvc-73c5d644-ddb1-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-64,Bound,pvc-3c2345e4-dd9f-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-192,Bound,pvc-51b672a8-ddb3-11e6-98ef-42010af000c3,10Gi,RWO,21d\nclaim-redacted-814,Bound,pvc-2b83ae02-dee0-11e6-98ef-42010af000c3,10Gi,RWO,20d\nclaim-redacted-775,Bound,pvc-ec1d47be-deb0-11e6-98ef-42010af000c3,10Gi,RWO,20d\nclaim-redacted-954,Bound,pvc-5a0acdc5-e3ee-11e6-98ef-42010af000c3,10Gi,RWO,13d\n\n\n\ndf = pd.DataFrame.from_csv('pvcs.csv')\ndf.head()\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\n\n\nNAME\n\n\n\n\n\n\n\n\n\nclaim-redacted-205\nBound\npvc-5643aadf-ddb3-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\nclaim-redacted-1088\nBound\npvc-bc68d752-ecba-11e6-98ef-42010af000c3\n10Gi\nRWO\n2d\n\n\nclaim-redacted-389\nBound\npvc-5d67d43e-ddcb-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\nclaim-redacted-119\nBound\npvc-73c5d644-ddb1-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\nclaim-redacted-64\nBound\npvc-3c2345e4-dd9f-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\n\n\n\n\n\n\n\n\ndf['username'] = np.array(df.index.str.extract('\\w+-([\\w-]+)-\\d+$'))\ndf['id'] = np.array(df.index.str.extract('\\w+-\\w+-(\\d+)$'))\ndf.head()\n\n/Users/redacted/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:1: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n  if __name__ == '__main__':\n/Users/redacted/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:2: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n  from ipykernel import kernelapp as app\n\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\nusername\nid\n\n\n\nNAME\n\n\n\n\n\n\n\n\n\n\n\n\nclaim-redacted-205\nBound\npvc-5643aadf-ddb3-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n\n205\n\n\nclaim-redacted-1088\nBound\npvc-bc68d752-ecba-11e6-98ef-42010af000c3\n10Gi\nRWO\n2d\nredacted\n1088\n\n\n\nclaim-redacted-389\nBound\npvc-5d67d43e-ddcb-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n389\n\n\n\nclaim-redacted-119\nBound\npvc-73c5d644-ddb1-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n119\n\n\n\nclaim-redacted-64\nBound\npvc-3c2345e4-dd9f-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n64\n\n\n\n\n\n\n\n\n\ndf.tail()\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\nusername\nid\n\n\nNAME\n\n\n\n\n\n\n\n\n\n\n\nclaim-redacted-562\nBound\npvc-4418d1d6-de0a-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n562\n\n\nclaim-redacted-400\nBound\npvc-2399fe54-ddcf-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n400\n\n\nclaim-redacted-946\nBound\npvc-294cb2c7-e35e-11e6-98ef-42010af000c3\n10Gi\nRWO\n14d\nredacted\n946\n\n\nclaim-redacted-900\nBound\npvc-f6859607-e1c0-11e6-98ef-42010af000c3\n10Gi\nRWO\n16d\nredacted\n900\n\n\nhub-db-dir\nBound\npvc-ea8a0bef-dd44-11e6-98ef-42010af000c3\n10Gi\nRWO\n22d\nNaN\nNaN\n\n\n\n\n\n\n\n\nvalids = df[~df['AGE'].str.contains('m')].dropna()\nvalids = valids[valids['id'] != '14']\nvalids.head()\n\n\n\n\n\n\n\nSTATUS\nVOLUME\nCAPACITY\nACCESSMODES\nAGE\nusername\nid\n\n\nNAME\n\n\n\n\n\n\n\n\n\n\n\nclaim-redacted-205\nBound\npvc-5643aadf-ddb3-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n205\n\n\nclaim-redacted-1088\nBound\npvc-bc68d752-ecba-11e6-98ef-42010af000c3\n10Gi\nRWO\n2d\nredacted\n1088\n\n\nclaim-redacted-389\nBound\npvc-5d67d43e-ddcb-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n389\n\n\nclaim-redacted-119\nBound\npvc-73c5d644-ddb1-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n119\n\n\nclaim-redacted-64\nBound\npvc-3c2345e4-dd9f-11e6-98ef-42010af000c3\n10Gi\nRWO\n21d\nredacted\n64\n\n\n\n\n\n\n\n\nvalids['username'].head().value_counts()\n\nredacted              1\nredacted    1\nredacted              1\nredacted            1\nredacted         1\nName: username, dtype: int64\n\n\n\nvalids['id'].value_counts().head()\n\n400     1\n90      1\n756     1\n488     1\n1029    1\nName: id, dtype: int64\n\n\n\nlen(valids['id'])\n\n1105\n\n\n\nimport sqlite3\nconn = sqlite3.connect('jupyterhub.sqlite')\nc = conn.cursor()\n\n\nc.execute('PRAGMA TABLE_INFO({})'.format('users'))\n\nc.fetchall()\n\n[(0, 'id', 'INTEGER', 1, None, 1),\n (1, 'name', 'VARCHAR(1023)', 0, None, 0),\n (2, '_server_id', 'INTEGER', 0, None, 0),\n (3, 'admin', 'BOOLEAN', 0, None, 0),\n (4, 'last_activity', 'DATETIME', 0, None, 0),\n (5, 'cookie_id', 'VARCHAR(1023)', 0, None, 0),\n (6, 'state', 'TEXT', 0, None, 0),\n (7, 'auth_state', 'TEXT', 0, None, 0)]\n\n\n\nc.execute('SELECT * FROM users LIMIT 20')\ncurrent_users = c.fetchall()\ncurrent_users[:2]\n\n[(1,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  '8ec09f8def774b668e838d1e70e00329',\n  None,\n  None),\n (2,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.938610',\n  'c98f59afc1fe4a70b18b451ac1190d45',\n  None,\n  None)]\n\n\n\nadmins = {user[1] for user in current_users if user[3] == 1}\nadmins\n\n{'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted',\n 'redacted'}\n\n\n\n# I just did this manually in the sqlite3 CLI\n# c.execute('DROP FROM users where id &gt; -1')\n# c.fetchall()\n\n\n---------------------------------------------------------------------------\nOperationalError                          Traceback (most recent call last)\n&lt;ipython-input-73-a8eb9e4f2c5d&gt; in &lt;module&gt;()\n----&gt; 1 c.execute('DROP FROM users where id &gt; -1')\n      2 c.fetchall()\n\nOperationalError: near \"FROM\": syntax error\n\n\n\n\nimport datetime\nimport itertools\n\nrecords = list(zip(\n    valids['id'].astype(int),\n    valids['username'],\n    itertools.repeat('NULL'),\n    [1 if name in admins else 0 for name in valids['username']],\n#     itertools.repeat('NULL'), # Pick a random valid value\n    itertools.repeat('2017-02-09 09:07:03.936620'), # Pick a random valid value\n    itertools.repeat('thisisadummycookiehopefullyitworks'),\n    itertools.repeat('NULL'),\n    itertools.repeat('NULL')\n))\nrecords[:3]\n\n[(205,\n  'redacted',\n  'NULL',\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  'NULL',\n  'NULL'),\n (1088,\n  'redacted',\n  'NULL',\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  'NULL',\n  'NULL'),\n (389,\n  'redacted',\n  'NULL',\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  'NULL',\n  'NULL')]\n\n\n\n# I hate everything\ndef record_to_sql(record):\n    return \"INSERT INTO users VALUES({}, '{}', {}, '{}', '{}', '{}', {}, {})\".format(\n        *record\n    )\n\nsql_statements = [record_to_sql(record) for record in records]\nrecord_to_sql(records[0])\n\n\"INSERT INTO users VALUES(205, 'redacted', NULL, '0', '2017-02-09 09:07:03.936620', 'thisisadummycookiehopefullyitworks', NULL, NULL)\"\n\n\n[(0, 'id', 'INTEGER', 1, None, 1),\n (1, 'name', 'VARCHAR(1023)', 0, None, 0),\n (2, '_server_id', 'INTEGER', 0, None, 0),\n (3, 'admin', 'BOOLEAN', 0, None, 0),\n (4, 'last_activity', 'DATETIME', 0, None, 0),\n (5, 'cookie_id', 'VARCHAR(1023)', 0, None, 0),\n (6, 'state', 'TEXT', 0, None, 0),\n (7, 'auth_state', 'TEXT', 0, None, 0)]\n\nwith conn:\n    c = conn.cursor()\n    for statement in sql_statements:\n        c.execute(statement)\n\n\nc.execute('SELECT * FROM users LIMIT 20')\nc.fetchall()\n\n[(2,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (3,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (4,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (5,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (6,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (7,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (8,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (9,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (10,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (11,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (12,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (13,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (15,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (17,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (18,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (19,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (20,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (21,\n  'redacted',\n  None,\n  1,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (22,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None),\n (24,\n  'redacted',\n  None,\n  0,\n  '2017-02-09 09:07:03.936620',\n  'thisisadummycookiehopefullyitworks',\n  None,\n  None)]"
-  },
-  {
-    "objectID": "incidents/2017-10-19-course-subscription-canceled.html",
-    "href": "incidents/2017-10-19-course-subscription-canceled.html",
-    "title": "Billing confusion with Azure portal causes summer hub to be lost",
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html",
+    "title": "Hubs throwing 505 errors",
     "section": "",
-    "text": "On October 10, 2017, the cloud vendor notified ds-instr that the data8r-17s subscription was canceled due to its end date, and we had 90 days to reactivate it using the educator portal. A support ticket was created to reverse the cancellation since the educator portal did not permit reactivation. On October 18 we were notified that the subscription’s resources were deleted.\nCoincidentally, a script was written on Oct. 9 to backup data to ds-instr’s Google Drive and it was performed for the instructor as a test. Unfortunately it wasn’t run for all users before the resources were taken offline."
+    "text": "PR 1 and PR 2 were merged to prod between 2 AM and 2.30 AM PST on 1/20. Difference due to the commits can be viewed here\nDue to these changes, image rebuild happened which broke multiple hubs which used that image including Datahub, ISchool, R, Data 100 and Data 140 hubs.\nOne of the dependencies highlighted as part of the image build had an upgrade which resulted in R hub throwing 505 error and Data 100/140 hub throwing “Error starting Kernel”. [Yuvi to fill in the right technical information]\nUser Impact:\n\n\nR Hub was not accessible for about 6 hours. Issue affected 10+ Stat 20 GSIs planning for their first class of the semester (catering to the needs of 600+ students). Hub went down for few minutes during the instruction.\nProb 140 hub was not available till 12.15 AM PST\nData 100 hub was not available till 12.33 AM. Thankfully, assignments were not due till friday (1/21)\nFew users in Ischool were affected as they could not access R Studio"
   },
   {
-    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#summary",
-    "href": "incidents/2017-10-19-course-subscription-canceled.html#summary",
-    "title": "Billing confusion with Azure portal causes summer hub to be lost",
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#summary",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#summary",
+    "title": "Hubs throwing 505 errors",
     "section": "",
-    "text": "On October 10, 2017, the cloud vendor notified ds-instr that the data8r-17s subscription was canceled due to its end date, and we had 90 days to reactivate it using the educator portal. A support ticket was created to reverse the cancellation since the educator portal did not permit reactivation. On October 18 we were notified that the subscription’s resources were deleted.\nCoincidentally, a script was written on Oct. 9 to backup data to ds-instr’s Google Drive and it was performed for the instructor as a test. Unfortunately it wasn’t run for all users before the resources were taken offline."
-  },
-  {
-    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#timeline",
-    "href": "incidents/2017-10-19-course-subscription-canceled.html#timeline",
-    "title": "Billing confusion with Azure portal causes summer hub to be lost",
-    "section": "Timeline",
-    "text": "Timeline\n\n2017-10-10 9:06a\nds-instr received an email from the cloud vendor:\n\nThe following subscriptions under your [cloud vendor] sponsorships for ds-instr@berkeley.edu have recently become canceled. Because these subscription(s) are canceled, all services have been suspended but no data has been lost. You have 90 days from the date of cancellation before [the cloud vendor] will delete the subscription and all attached data. Please use the Educator Portal to reactivate the subscription(s).\n\n\n\n\nSubscription Name\nSubscription Id\nCanceled Reason\n\n\n\n\ndata8r-17s\nomitted here\nSubscription End Date\n\n\n\n\n\n9:30\nThe instructor was notified. The educator portal did not provide a way to view or alter the subscription end date of a canceled subscription so a support request was filed at the cloud vendor.\n\n\n11:14\nThe cloud vendor asks that a payment instrument be added to the ds-instr account. We respond that the account is funded by a sponsorship.\n\n\n17:22\nThe cloud vendor contacts their sponsorship team.\n\n\n2017-10-11 15:00\nThe cloud vendor calls to discuss the situation. Screenshots of the educator and cloud portal were sent to the cloud vendor.\n\n\n2017-10-12 16:19\nThe cloud vendor offers to enable the subscription for a 60 minute period from the backend so that the End Date may be extended from the portal. Though the subscription is re-enabled for an hour, the portal still does not permit the subscription parameters to be changed.\n\n\n2017-10-18 15:29\nThe cloud vendor says that the subscription was actually disabled because it had exhausted the allocated funds, and the data was deleted within 24 hours despite the stated 90 day grace period. Later the following was provided by the cloud vendor:\n\nI worked with our backend engineering team last night and I am afraid to say that we could not retrieve the storage account after all our sincere efforts. I understand how frustrating it would be for you and I do not have the words to express the same, I just wish if I could be of some help to you.\nHaving said that we did dig into the reasons behind this situation, the subscription was initially suspended by an internal engineering job occurred that auto-suspended all Academic Account Sponsorship subscriptions with an end date that was part of the previous fiscal year. Usually this suspension does not delete the subscription. There are a few [cloud vendor] accounts which are on legacy commerce platform which are affected and these accounts are in the process of modern platform. Your account was in the transition mode when the subscription got suspended and your account was partially converted to the modern platform. The billing & subscription part was converted to the modern platform but the not at the service level. Hence you got the message that your data would be retained for 90 days, at the same stated at the service level it was not converted to the modern hence the data got deleted.\nI had a detailed discussion with our product group team on this and how we can avoid this in future. First of all, your account is now completely migrated/transitioned completely to the modern platform. Also, to ensure that our other Academic Account Sponsorships customers do not face the same issue they have agreed to complete the migration manually on those accounts.\n\n\n\n2017-10-19 10:53\nThe cloud vendor compensates ds-instr with an additional $10k for the experience."
-  },
-  {
-    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#conclusion",
-    "href": "incidents/2017-10-19-course-subscription-canceled.html#conclusion",
-    "title": "Billing confusion with Azure portal causes summer hub to be lost",
-    "section": "Conclusion",
-    "text": "Conclusion\nThere were insufficient funds on the subscription to persist its resources. The resources were deleted by the cloud vendor before the grace period ran out."
-  },
-  {
-    "objectID": "incidents/2017-10-19-course-subscription-canceled.html#action-items",
-    "href": "incidents/2017-10-19-course-subscription-canceled.html#action-items",
-    "title": "Billing confusion with Azure portal causes summer hub to be lost",
-    "section": "Action items",
-    "text": "Action items\n\nProcess\n\nUntil there is a per-user backup implemented hub-side, set a schedule for backing up user data for every course.\nAlways set a billing alert at some conservative amount less than the subscription alotment\nIf a subscription is ever canceled, backup user data within 24 hours, regardless of the stated grace period."
+    "text": "PR 1 and PR 2 were merged to prod between 2 AM and 2.30 AM PST on 1/20. Difference due to the commits can be viewed here\nDue to these changes, image rebuild happened which broke multiple hubs which used that image including Datahub, ISchool, R, Data 100 and Data 140 hubs.\nOne of the dependencies highlighted as part of the image build had an upgrade which resulted in R hub throwing 505 error and Data 100/140 hub throwing “Error starting Kernel”. [Yuvi to fill in the right technical information]\nUser Impact:\n\n\nR Hub was not accessible for about 6 hours. Issue affected 10+ Stat 20 GSIs planning for their first class of the semester (catering to the needs of 600+ students). Hub went down for few minutes during the instruction.\nProb 140 hub was not available till 12.15 AM PST\nData 100 hub was not available till 12.33 AM. Thankfully, assignments were not due till friday (1/21)\nFew users in Ischool were affected as they could not access R Studio"
   },
   {
-    "objectID": "incidents/2018-06-11-course-subscription-canceled.html",
-    "href": "incidents/2018-06-11-course-subscription-canceled.html",
-    "title": "Azure billing issue causes downtime",
-    "section": "",
-    "text": "On June 11, 2018, the cloud vendor notified ds-instr that the data8-17f-prod subscription was canceled due to its usage cap. The educator portal confirmed that the spend had surpassed the budget. After additional funds were allocated to the subscription, a portion of the VMs were manually started. The hub came back online after pods were forcibly deleted and nodes were cordoned."
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#hub-information",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#hub-information",
+    "title": "Hubs throwing 505 errors",
+    "section": "Hub information",
+    "text": "Hub information\n\nHub URL: https://r.datahub.berkeley.edu/ and most other hubs highlighted above"
   },
   {
-    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#summary",
-    "href": "incidents/2018-06-11-course-subscription-canceled.html#summary",
-    "title": "Azure billing issue causes downtime",
-    "section": "",
-    "text": "On June 11, 2018, the cloud vendor notified ds-instr that the data8-17f-prod subscription was canceled due to its usage cap. The educator portal confirmed that the spend had surpassed the budget. After additional funds were allocated to the subscription, a portion of the VMs were manually started. The hub came back online after pods were forcibly deleted and nodes were cordoned."
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#timeline-if-relevant",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#timeline-if-relevant",
+    "title": "Hubs throwing 505 errors",
+    "section": "Timeline (if relevant)",
+    "text": "Timeline (if relevant)\n\n{{ 2022-01-20 Between 02:00 and 02.30 PM }}\nPR 1 and PR 2 were merged to prod. Notably, PR 1 had multiple commits related to creation of Stat 20 hub, Stat 259 hub etc..\n\n\n{{ 06:10 }}\nAndrew Bray (Stat 20 instructor) raised a github issue around 5.45 AM PST.\n\n\n{{ 07:45 }}\nYuvi quickly jumped in to make a fix to get the R hub working. However this fix resulted in breaking Stat 20 hub.\n\n\n{{ 07:53 }}\nISchool folks reported issues with using RStudio in Datahub\n\n\n{{ 08:45 }}\nYuvi fixed issue with Stat 20 and other hubs\n\n\n{{ 12:10 }}\nGSIs from Data 100 and 140 reported “Unhandled error” in their hubs\n\n\n{{ 12:15 }}\nGSIs for Data 140 hub reported that the error was fixed\n\n\n{{ 12:33 }}\nGSIs Data 100 hub reported that the error was fixed"
   },
   {
-    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#timeline",
-    "href": "incidents/2018-06-11-course-subscription-canceled.html#timeline",
-    "title": "Azure billing issue causes downtime",
-    "section": "Timeline",
-    "text": "Timeline\n\n2018-06-11 9:02a\nds-instr received an email from the cloud vendor:\n\nThe following subscriptions under your Microsoft Azure sponsorships for ds-instr@berkeley.edu have recently become canceled. Because these subscription(s) are canceled, all services have been suspended but no data has been lost. You have 90 days from the date of cancellation before Microsoft will delete the subscription and all attached data. Please use the Educator Portal to reactivate the subscription(s).\n\n\n\n\nSubscription Name\nSubscription Id\nCanceled Reason\n\n\n\n\ndata8-17f-prod\nomitted here\nSubscription Cap\n\n\n\n\n\n9:29\nThe subscription status was confirmed at https://www.microsoftazuresponsorships.com/Manage. In order to allocate additional budget to data8-17f-prod, budget for other subscriptions had to be reduced.\n\n\n9:40\nVMs were turned on at https://portal.azure.com: 3 nodes in each node pool, the nfs server, the kubernetes master, and the database server.\n\n\n9:45\nThe hub was unreachable even though the VMs were online. The hub and proxy pods were shown as Running and all nodes were shown as online even though some nodes had not been started. The offline cluster nodes were manually cordoned. All pods had to be forcibly deleted before they would start.\n\n\n10:14\nThe Billing Alert Service was checked at https://account.azure.com/Subscriptions/alert?subscriptionId=06f94ac5-b029-411f-8896-411f3c6778b4 and it was discovered that alerts were no longer registered."
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#what-went-wrong",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#what-went-wrong",
+    "title": "Hubs throwing 505 errors",
+    "section": "What went wrong",
+    "text": "What went wrong\n\nR, Stat 20, Datahub, ISchool, Data 100 and 140 hubs went down around 2.30 AM PST. However, the team was aware of these issues only when users reported errors at different time intervals (as listed above)\nMultiple commits went through a single PR. Dependency package’s version upgrade broke the image build (Yuvi to fill in the required details)\n\nThings that could have gone better. Ideally these should result in concrete action items that have GitHub issues created for them and linked to under Action items."
   },
   {
-    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#conclusion",
-    "href": "incidents/2018-06-11-course-subscription-canceled.html#conclusion",
-    "title": "Azure billing issue causes downtime",
-    "section": "Conclusion",
-    "text": "Conclusion\nThere were insufficient funds on the subscription to persist its resources. The subscription budget was increased and the hub was brought back online. The billing alert service that was configured to prevent such incidents did not function properly."
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#where-we-got-lucky",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#where-we-got-lucky",
+    "title": "Hubs throwing 505 errors",
+    "section": "Where we got lucky",
+    "text": "Where we got lucky\nThese are good things that happened to us but not because we had planned for them.\n\nYuvi was awake at the time when issue was reported and was able to fix the issues immediately.\nClasses using hubs were not completely affected due to this outage (Data 100 did not have assignments due till 1/21 and Stat 20 had few mins of outage during instruction)"
   },
   {
-    "objectID": "incidents/2018-06-11-course-subscription-canceled.html#action-items",
-    "href": "incidents/2018-06-11-course-subscription-canceled.html#action-items",
-    "title": "Azure billing issue causes downtime",
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#action-items",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#action-items",
+    "title": "Hubs throwing 505 errors",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nDo not use subscription portal billing alerts.\nManually check subscription usage via an unattended process."
+    "text": "Action items\nThese are only sample subheadings. Every action item should have a GitHub issue (even a small skeleton of one) attached to it, so these do not get forgotten. These issues don’t have to be in infrastructure/, they can be in other repositories.\n\nProcess/Policy improvements\n\n{{Develop manual testing process whenever a PR gets merged to staging of the major hubs (till automated test suites are written)}} link to github issue]\nDevelop a policy around when to create a new hub and what type of changes get deployed to Datahub!\n\n\n\nDocumentation improvements\n\n{{ Start writing after action reports for future outages }} [link to github issue]\n{{ summary }} [link to github issue]\n\n\n\nTechnical improvements\n\n{{ Enabling logging mechanism across all hubs to track future outages }}\n{{ Adapt 2I2C testing suite to develop automated test cases that check the sanity of the different services whenever a PR gets merged in staging}} [link to github issue]\n{{ Investigate the reason why pager duty did not throw an alert for 5xx errors when the hubs went down. Fix the alerting mechanism so that they notify all kind of errors }} [link to github issue]\n{{ Adding R Studio as part of Repo2Docker}} [link to github issue]"
   },
   {
-    "objectID": "incidents/2024-core-node-incidents.html",
-    "href": "incidents/2024-core-node-incidents.html",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
-    "section": "",
-    "text": "Over the past couple of years, all of our production hubs have been having persistent issues with our core nodes having major load spikes during ‘peak’ usage and the impacted node (which hosts all of our hub and proxy pods – not user pods) crashing. This would then impact every hub, causing all users to see 503 http errors until a new node finishing spinning up. We also suspect that the ‘white screen’ issue some users see after logging in is related to this.\nThese outages would usually last anywhere from 45 to 90+ minutes. The first chunk of time would be the core node getting wedged and eventually dying, and the last 15-20 minutes would be spent on the new node spinning up and services restarting.\nMany of these incidents are tracked here.\nWe have spent much time working to debug and track this, including with our friends at 2i2c. After much deep-diving and debugging, we were able to narrow this down to a memory (socket?) leak in the configurable http proxy.\nAfter some back and forth w/the upstream maintainers, we received a forked version of the proxy to test.\nDuring this testing, we triggered some user-facing downtime, as well as the proxy itself crashing and causing small outages.\nAnother (unrelated) issue that impacted users was that GKE was autoscaling our core pool (where the hub and proxy pods run) node to zero. Since it takes about 10-15m for a new node to spin up, all hubs were inaccessible until the new node was deployed.\nUser Impact:\n\n\nOn the afternoon of Feb 7th, I was testing the fork of the proxy + some revised timeouts on the Data 8 hub. This caused the proxy to crash every ~20m over the course of a few hours. I then reverted the fork and timeout changes.\nDuring the latter half of February, our core pool was being autoscaled from 1 to 0 nodes. This caused multiple short outages.\nThe proxy pods for Data 8 and Data 100 (our largest classes) crashed continually under load (~250+ simultaneous users), causing users to receive 500 HTTP errors until the pod automatically restarted."
+    "objectID": "incidents/2022-01-20-package-dependency-upgrade-incident.html#actions",
+    "href": "incidents/2022-01-20-package-dependency-upgrade-incident.html#actions",
+    "title": "Hubs throwing 505 errors",
+    "section": "Actions",
+    "text": "Actions\n\nIncident has been dealt with or is over\nSections above are filled out\nIncident title and after-action report is cleaned up\nAll actionable items above have linked GitHub Issues"
   },
   {
-    "objectID": "incidents/2024-core-node-incidents.html#summary",
-    "href": "incidents/2024-core-node-incidents.html#summary",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
+    "objectID": "incidents/2017-05-09-gce-billing.html",
+    "href": "incidents/2017-05-09-gce-billing.html",
+    "title": "Oops we forgot to pay the bill",
     "section": "",
-    "text": "Over the past couple of years, all of our production hubs have been having persistent issues with our core nodes having major load spikes during ‘peak’ usage and the impacted node (which hosts all of our hub and proxy pods – not user pods) crashing. This would then impact every hub, causing all users to see 503 http errors until a new node finishing spinning up. We also suspect that the ‘white screen’ issue some users see after logging in is related to this.\nThese outages would usually last anywhere from 45 to 90+ minutes. The first chunk of time would be the core node getting wedged and eventually dying, and the last 15-20 minutes would be spent on the new node spinning up and services restarting.\nMany of these incidents are tracked here.\nWe have spent much time working to debug and track this, including with our friends at 2i2c. After much deep-diving and debugging, we were able to narrow this down to a memory (socket?) leak in the configurable http proxy.\nAfter some back and forth w/the upstream maintainers, we received a forked version of the proxy to test.\nDuring this testing, we triggered some user-facing downtime, as well as the proxy itself crashing and causing small outages.\nAnother (unrelated) issue that impacted users was that GKE was autoscaling our core pool (where the hub and proxy pods run) node to zero. Since it takes about 10-15m for a new node to spin up, all hubs were inaccessible until the new node was deployed.\nUser Impact:\n\n\nOn the afternoon of Feb 7th, I was testing the fork of the proxy + some revised timeouts on the Data 8 hub. This caused the proxy to crash every ~20m over the course of a few hours. I then reverted the fork and timeout changes.\nDuring the latter half of February, our core pool was being autoscaled from 1 to 0 nodes. This caused multiple short outages.\nThe proxy pods for Data 8 and Data 100 (our largest classes) crashed continually under load (~250+ simultaneous users), causing users to receive 500 HTTP errors until the pod automatically restarted."
+    "text": "On May 9, 2017, the compute resources associated with the data-8 project at GCE were suspended. All hubs including datahub, stat28, and prob140 were not reachable. This happened because the grant that backed the project’s billing account ran out of funds. The project was moved to a different funding source and the resources gradually came back online."
   },
   {
-    "objectID": "incidents/2024-core-node-incidents.html#hub-information",
-    "href": "incidents/2024-core-node-incidents.html#hub-information",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
-    "section": "Hub information",
-    "text": "Hub information\n\nData 8\nData 100"
+    "objectID": "incidents/2017-05-09-gce-billing.html#summary",
+    "href": "incidents/2017-05-09-gce-billing.html#summary",
+    "title": "Oops we forgot to pay the bill",
+    "section": "",
+    "text": "On May 9, 2017, the compute resources associated with the data-8 project at GCE were suspended. All hubs including datahub, stat28, and prob140 were not reachable. This happened because the grant that backed the project’s billing account ran out of funds. The project was moved to a different funding source and the resources gradually came back online."
   },
   {
-    "objectID": "incidents/2024-core-node-incidents.html#timeline",
-    "href": "incidents/2024-core-node-incidents.html#timeline",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
+    "objectID": "incidents/2017-05-09-gce-billing.html#timeline",
+    "href": "incidents/2017-05-09-gce-billing.html#timeline",
+    "title": "Oops we forgot to pay the bill",
     "section": "Timeline",
-    "text": "Timeline\n\n2024-03-05 Data8 outage\nbetween ~4pm and ~5:15pm, data8’s configurable-http-proxy (chp) was oomkilled and caused many 503 errors to be issued to the users. here’s a rough timeline of what happened (culled from grafana, gcp logs and kernel logs on the core node):\n\n\n15:12:00\n~280 concurrent users\n\n\n15:12:42\nchp “uncaught exception: write EPIPE”\n\n\n16:00:00\nproxy ram 800Mi (steady)\n\n\n16:05:00\n~300 concurrent users\n\n\n~16:05:00\nspike on proxy — cpu 181%, mem 1.06Gi –&gt; 1.86Gi\n\n\n16:05:53\nchp healthz readiness probe failure\n\n\n16:05:56\nchp/javascript runs out of heap “Ineffective mark-compass near heap limit Allocation Failed”\n\n\n16:05:57-58\nchp restarts\n\n\n16:05:57\nnode dmesg: “TCP: request_sock_TCP: Possible SYN flooding on port 8000. Sending cookies. Check SNMP counters.”\n\n\n~16:06:00\n2.5K 503 errors\n\n\n~16:06:00 - 16:16:00\nmany many chp routes added and deleted causing 503 errors for some users\n\n\n16:16:00 - 16:49:00\neverything back to normal\n\n\n16:49:00\n~300 users (slowly decreasing)\n\n\n~16:49:00\nspike on proxy — cpu 107%, mem 814Mi –&gt; 2.45Gi\n\n\n16:49:40\ncore node dmesg: node invoked oom-killer: gfp_mask=0xcc0(GFP_KERNEL), order=0, oom_score_adj=999\n\n\n16:50:25\nchp restarts (no heap error)\n\n\n~16:50:00\n5.7K 503 errors\n\n\n16:54:15 - 17:15:31\n300 users (slowly decreasing), 3x chp “uncaught exception: write EPIPE”, intermittent 503 errors in spikes of 30, 60, 150, hub latency 2.5sec\n\n\n18:47:19 - 18:58:10\n~120 users (constant), 3x chp “uncaught exception: write EPIPE”, intermittent 503 errors in spikes of 30, 60, hub latency 3sec\n\n\n~19:00\nthings return to normal"
-  },
-  {
-    "objectID": "incidents/2024-core-node-incidents.html#what-went-wrong",
-    "href": "incidents/2024-core-node-incidents.html#what-went-wrong",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
-    "section": "What went wrong",
-    "text": "What went wrong\n\nOnly being able to test things like the forked chp in prod is dangerous and has a huge potential impact on users.\nThe configurable http proxy (chp) is written in javascript (nodejs), and under load (~250+ concurrent users) it leaks ports and eventually fills up the heap (~728M) and gets oomkilled.\nThe CPU allocation on the core node was also spiking, potentially causing login latency across all hubs.\nUpon chp restart, it can take up to 10-15m for the routing table to be repopulated. During this time most, if not all, users that were already connected to the hub will get 503 errors. Any new logins during this time will not."
+    "text": "Timeline\n\n2017-05-09 16:51\nA report in the Data 8 Spring 2017 Staff slack, #jupyter channel, says that datahub is down. This is confirmed. Attempting to access the provisioner via gcloud compute ssh provisioner-01 fails with:\nERROR: (gcloud.compute.ssh) Instance [provisioner-01] in zone [us-central1-a] has not been allocated an external IP address yet. Try rerunning this command later.\n\n\n17:01\nThe Google Cloud console shows that the billing account has run out of the grant that supported the data-8 project. The project account is moved to another billing account which has resources left.\nThe billing state is confirmed by gcloud messages:\nGoogle Compute Engine: Project data-8 cannot accept requests to setMetadata while in an inactive billing state.  Billing state may take several minutes to update.\n\n\n17:09\nprovisioner-01 is manually started. All pods in the datahub namespace are deleted.\n\n\n17:15\ndatahub is back online. stat28 and prob140 hub pods are manually killed. After a few moments the hubs are back online. The autoscaler is started.\n\n\n17:19\nThe slack duplicator is started.\n\n\n2017-05-10 10:48\nA report in uc-jupyter #jupyterhub says that try.datahub is down. This is confirmed and the hub in the tmp namespace is killed. The hub comes online a couple of minutes later."
   },
   {
-    "objectID": "incidents/2024-core-node-incidents.html#where-we-got-lucky",
-    "href": "incidents/2024-core-node-incidents.html#where-we-got-lucky",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
-    "section": "Where we got lucky",
-    "text": "Where we got lucky\n\nSince we bumped the RAM allocated to the core node from 8G to 32G instances like are isolated to whatever hub’s chp begins failing, and does not impact all hubs."
+    "objectID": "incidents/2017-05-09-gce-billing.html#conclusion",
+    "href": "incidents/2017-05-09-gce-billing.html#conclusion",
+    "title": "Oops we forgot to pay the bill",
+    "section": "Conclusion",
+    "text": "Conclusion\nThere was insufficient monitoring of the billing status."
   },
   {
-    "objectID": "incidents/2024-core-node-incidents.html#action-items",
-    "href": "incidents/2024-core-node-incidents.html#action-items",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
+    "objectID": "incidents/2017-05-09-gce-billing.html#action-items",
+    "href": "incidents/2017-05-09-gce-billing.html#action-items",
+    "title": "Oops we forgot to pay the bill",
     "section": "Action items",
-    "text": "Action items\n\nProcess/Policy improvements\n\nWork with instructors to identify and immediately troubleshoot directly with users impacted by the white screen issue.\nSpin up a dev hub and figure out how to use hubtraf to simulate a large number of users doing work.\n\n\n\nDocumentation improvements\nNone.\n\n\nTechnical improvements\n\nContinue to track the port leak issue upstream.\nDeploy a new core pool with the same RAM and more CPU (Jira DH-259).\nSpin up a dev hub and figure out how to use hubtraf to simulate a large number of users doing work."
-  },
-  {
-    "objectID": "incidents/2024-core-node-incidents.html#actions",
-    "href": "incidents/2024-core-node-incidents.html#actions",
-    "title": "Core nodes being autoscaled, configurable HTTP proxy crashes",
-    "section": "Actions",
-    "text": "Actions\n\nIncident has been dealt with or is over\nSections above are filled out\nIncident title and after-action report is cleaned up\nAll actionable items above have linked GitHub Issues"
+    "text": "Action items\n\nProcess\n\nIdentify channels for billing alerts.\nIdentify billing threshold functions that predict when funds will run out.\nEstablish off-cloud backups. The plan is to do this via nbgdrive.\nStart autoscaler automatically. It is manually started at the moment.\n\n\n\nMonitoring\n\nSetup scheduled billing reports and threshold alarms.\nSetup hub monitoring!\nThe slack duplicator runs in one of the GCP clusters. When the clusters go down, slack messages aren’t forwarded from the data8-sp17-staff slack to uc-jupyter."
   },
   {
-    "objectID": "incidents/2017-02-09-datahub-db-outage.html",
-    "href": "incidents/2017-02-09-datahub-db-outage.html",
-    "title": "JupyterHub db manual overwrite",
+    "objectID": "incidents/2018-02-06-hub-db-dir.html",
+    "href": "incidents/2018-02-06-hub-db-dir.html",
+    "title": "Azure PD refuses to detach, causing downtime for data100",
     "section": "",
-    "text": "Datahub was reportedly down at 1am. Users attempting to log in to datahub were greeted with a proxy error. The hub pod was up but the log was full of sqlite errors. After the hub pod was deleted and a new one came up, students logging in to datahub found their notebooks were missing and their home directories were empty. Once this was fixed, some students still were being logged in as a different particular user. Finally, students with a ‘.’ in their username were still having issues after everyone else was fine. This was all fixed and an all-clear signalled at about 2017-02-09 11:35 AM."
+    "text": "On February 5, 2018, a PR was merged into the production cluster for Data 100. The CI got as far as running helm upgrade but the hub’s persistent volume would not detach from the old hub. The new hub pod had to wait on the hub-db-dir volume and so would not start. The persistent volume claim was ultimately deleted. The subsequent helm upgrade created a new volume and a new hub pod was able to start."
   },
   {
-    "objectID": "incidents/2017-02-09-datahub-db-outage.html#summary",
-    "href": "incidents/2017-02-09-datahub-db-outage.html#summary",
-    "title": "JupyterHub db manual overwrite",
+    "objectID": "incidents/2018-02-06-hub-db-dir.html#summary",
+    "href": "incidents/2018-02-06-hub-db-dir.html#summary",
+    "title": "Azure PD refuses to detach, causing downtime for data100",
     "section": "",
-    "text": "Datahub was reportedly down at 1am. Users attempting to log in to datahub were greeted with a proxy error. The hub pod was up but the log was full of sqlite errors. After the hub pod was deleted and a new one came up, students logging in to datahub found their notebooks were missing and their home directories were empty. Once this was fixed, some students still were being logged in as a different particular user. Finally, students with a ‘.’ in their username were still having issues after everyone else was fine. This was all fixed and an all-clear signalled at about 2017-02-09 11:35 AM."
+    "text": "On February 5, 2018, a PR was merged into the production cluster for Data 100. The CI got as far as running helm upgrade but the hub’s persistent volume would not detach from the old hub. The new hub pod had to wait on the hub-db-dir volume and so would not start. The persistent volume claim was ultimately deleted. The subsequent helm upgrade created a new volume and a new hub pod was able to start."
   },
   {
-    "objectID": "incidents/2017-02-09-datahub-db-outage.html#timeline",
-    "href": "incidents/2017-02-09-datahub-db-outage.html#timeline",
-    "title": "JupyterHub db manual overwrite",
+    "objectID": "incidents/2018-02-06-hub-db-dir.html#timeline",
+    "href": "incidents/2018-02-06-hub-db-dir.html#timeline",
+    "title": "Azure PD refuses to detach, causing downtime for data100",
     "section": "Timeline",
-    "text": "Timeline\n\n2017-02-09 00:25 - 00:29 AM\nAttempting to debug some earlier 400 errors, Trying to set base_url and ip to something incorrect to see if it will cause a problem.\nkubectl exec hub-deployment-something --namespace=datahub -it bash\napt-get install sqlite3\nsqlite3\nATTACH 'jupyterhub.sqlite AS my_db;\nSELECT name FROM my_db.sqlite_master WHERE type='table';\nSELECT * FROM servers;\nSELECT * FROM servers WHERE base_url LIKE '%&lt;USER&gt;%';\nUPDATE servers SET ip='' WHERE base_url LIKE '%&lt;USER&gt;%';\nUPDATE servers SET base_url='/&lt;something-wrong&gt; WHERE base_url LIKE '%&lt;USER&gt;%';\nCtrl+D (exit back into bash shell)\nchecked datahub.berkeley.edu, and nothing happened to the account saw that the sql db was not updated, attempt to run .save\n```bash\nsqlite3\n.save jupyterhub.sqlite\nThis replaced the db with an empty one, since ATTACH was not run beforehand.\n\n\n0:25:59 AM\nFollowing exception shows up in hub logs:\nsqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: proxies [SQL: 'SELECT proxies.id AS proxies_id, proxies._public_server_id AS proxies__public_server_id, proxies._api_server_id AS proxies__api_server_id \\nFROM proxies \\nWHERE proxies.id = ?'] [parameters: (1,)]\nThis continues for hub table as well, since those two seem to be most frequently used.\n\n\n1:12 AM\nSam’s roommate notices that he can log in to datahub but all his notebooks are gone. We notice that there are only ~50 users on the JHub admin panel when there used to be ~1000, so we believe that this is because the JHub sqlite user database got wiped/corrupted, then created an account for his roommate when he logged in, then created a new persistent disk since it lost track of his old one.\nThis is confirmed soon after:\n$ kubectl --namespace=datahub get pvc | grep  &lt;username&gt;\nclaim-&lt;username&gt;-257     Bound     pvc-3b405e13-ddb4-11e6-98ef-42010af000c3   10Gi       RWO           21d\nclaim-&lt;username&gt;-51      Bound     pvc-643dd900-eea7-11e6-a291-42010af000c3   10Gi       RWO           5m\n\n\n1:28 AM\nWe shut down the hub pod by scaling the replicas to 0.\nWe then begin recreating the JHub sqlite database by taking the Kubernetes PVCs and matching them back with the user ids. We could do this because the name of the PVC contains a sanitized form of the username and the userid.\nHere’s the notebook that was used to recreate the db from PVCs: pvc-sqlite.ipynb 2017-02-09-datahub-db-outage-pvc-recreate-script.ipynb\n\n\n2:34 AM\nWe recreate the sqlite3 database. Initially each user’s cookie_id was set to a dummy cookie value.\n\n\n2:42 AM\nUser cookie_id values are changed to null rather than dummy value. The sqlite file is then attached back to datahub. The number of users shown on admin page is back to ~1000. The hub was up, and a spot check of starting other user’s servers seem to work. Some users get redirected to one particular user, but deleting and recreating the affected user seems to fix this.\n\n\n10:11 AM\nAttempt to log everyone out by changing cookie secret in hub pod at /srv/jupyterhub/jupyterhub_cookie_secret. Just one character near the end was changed, and pod restarted. No effect. One character at the beginning of secret was changed next, and restarted - this caused actual change, and logged all users out.\nPeople are still being redirected to one particular user’s account when they log in. More looking around required.\n\n\n10:17 AM\nJohn Denero advises students to use ds8.berkeley.edu right now. ds8.berkeley.edu promptly starts crashing because it does not have resources for a data8 level class.\n\n\n10:29 AM\nAll user pods are deleted, which finally properly logs everyone out. However, people logging in are still all getting the same user’s pods.\n\n\n10:36 AM\nNotice that cookie_id column in the user database table is empty for many users, and the user that everyone is being logged in as has an empty cookie_id too and is the ‘first’ on the table when sorted in ascending by id. Looking at the JupyterHub code, cookie_id is always supposed to be set to a uuid, and never supposed to be empty. Setting cookie_id for users fixes their issues, and seems to spawn them into their own notebook.\n\n\n10:45 AM\nA script is run that populates cookie_id for all users, and restarts the hub to make sure there’s no stale cache in RAM. All user pods are deleted again. Most users are back online now! More users start testing and confirming things are working for them.\n\n\n10:53 AM\nUser with a ‘.’ in their name reports that they’re getting an empty home directory. More investigation shows two users - one with a ‘.’ in their name that is newer, and one with a ‘-’ in their name instead of ‘.’ that is older. Hypothesis is that one of them is the ‘original’, but they’re all attaching to a new one that is empty. Looking at pvcs confirms this - there are two PVCs for users with a . in their name who have tried to log in, and they differ only by ids.\nThere is some confusion about users ending up on prob140, because the data8.org homework link is changed to use that temporarily.\n\n\n11:05 AM\nDirectly modifying the user table to rename the user with the ‘-’ in the name to have a ‘.’ seems to work for people.\n\n\n11:15 AM\nA script is run that modifies the database user table for all users with a ‘-’ in their name, and the ‘-’ is replaced with a ‘.’. The new users created with the ‘.’ in their name are dropped before this.\n\n\n11:17 AM\nAll clear given for datahub.berkeley.edu\n\n\n11:19 AM\nLocally verified that running .save  on sqlite3 will overwrite the db file without any confirmation, and is most likely cause of the issue. Conclusion Accidental overwriting of the sqlite file during routine debugging operation led all tables being deleted. Users were getting new user ids when they were logging in now, causing them to get new disks provisioned - and these disks were empty. During reconstruction of the db, cookie_id was missing for several users, causing them all to log in to one particular user’s notebook. Users with ‘.’ in their name were also set up slightly incorrectly - their pods have ‘-’ in them but the user name should have a ‘.’."
+    "text": "Timeline\n\n2018-02-05 20:57\nA PR for data100 is merged.\n\n\n21:20\nTowards the end of the build, the upgrade fails because the new hub pod does not start up. The hub-db-dir volume remains bound to the old hub pod which is stuck in a Terminating state. The hub pod only completes termination when delete is passed a grace period of 0. The hub volume remains bound however.\n\n\n21:30\nCI is restarted but by the time helm is run, the hub-db-dir volume remains bound and cannot be attached to the new hub pod. Additionally, helm errors because the jupyterhub-internal ingress object cannot be found even though it does exist.\n\n\n21:45\nSince it cannot be determined what node the volume is bound to, the volume is deleted. The jupyterhub-internal ingress object is also deleted prior to restarting the CI build.\n\n\n22:05\nThe hub comes up with a new hub-db-dir volume. CI fails due to the same jupyterhub-internal object error."
   },
   {
-    "objectID": "incidents/2017-02-09-datahub-db-outage.html#action-items",
-    "href": "incidents/2017-02-09-datahub-db-outage.html#action-items",
-    "title": "JupyterHub db manual overwrite",
+    "objectID": "incidents/2018-02-06-hub-db-dir.html#conclusion",
+    "href": "incidents/2018-02-06-hub-db-dir.html#conclusion",
+    "title": "Azure PD refuses to detach, causing downtime for data100",
+    "section": "Conclusion",
+    "text": "Conclusion\nAzure was not able to detach the hub-db-dir azure disk from the hub pod. The pvc is deleted and the hub comes up on the next CI run."
+  },
+  {
+    "objectID": "incidents/2018-02-06-hub-db-dir.html#action-items",
+    "href": "incidents/2018-02-06-hub-db-dir.html#action-items",
+    "title": "Azure PD refuses to detach, causing downtime for data100",
     "section": "Action items",
-    "text": "Action items\n\nUpstream bug reports for JupyterHub\n\nJupyterHub only uses a certain length of the cookie secret, and discards the rest. This causes confusion when trying to change it to log people out. Issue\nThe cookie_id column in the users table should have UNIQUE and NOT NULL constraints. Issue\n\n\n\nUpstream bug reports for KubeSpawner\n\nSupport using username hashes in PVC and Pod Names rather than user ids, so that pod and PVC names remain constant even when DB is deleted. Issue\n\n\n\nUpstream bug reports for OAuthenticator\n\nSupport setting id of user in user table to be same as ‘id’ provided by Google authenticator, thus providing a stable userid regardless of when the user first logged in. Issue\n\n\n\nDataHub deployment changes\n\nSwitch to using Google Cloud SQL, which provides hosted and managed MySQL database\nPerform regular and tested backups of the database\nStart writing an operational FAQ for things to do and not do\nSetup better monitoring and paging systems\nDocument escalation procedures explicitly"
+    "text": "Action items\n\nProcess\n\nStore the hub db in a cloud database to eliminate reliance on hub volume.\nDowngrade helm to 2.6.x to see if this fixes the helm upgrades."
   },
   {
-    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html",
-    "href": "incidents/2017-03-23-kernel-deaths-incident.html",
-    "title": "Weird upstream ipython bug kills kernels",
+    "objectID": "incidents/2018-02-28-hung-node.html",
+    "href": "incidents/2018-02-28-hung-node.html",
+    "title": "A node hangs, causing a subset of users to report issues",
     "section": "",
-    "text": "A seemingly unrelated change caused user kernels to die on start (making notebook execution impossible) for newly started user servers from about Mar 22 19:30 to Mar 23 09:45. Most users didn’t see any errors until start of class at about 9AM, since they were running servers that were previously started."
+    "text": "On February 28, 2018, a handful of users reported on piazza that there servers wouldn’t start. It was determined that all problematic servers were running on the same node. After the node was cordoned and rebooted, the student servers were able to start properly."
   },
   {
-    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#summary",
-    "href": "incidents/2017-03-23-kernel-deaths-incident.html#summary",
-    "title": "Weird upstream ipython bug kills kernels",
+    "objectID": "incidents/2018-02-28-hung-node.html#summary",
+    "href": "incidents/2018-02-28-hung-node.html#summary",
+    "title": "A node hangs, causing a subset of users to report issues",
     "section": "",
-    "text": "A seemingly unrelated change caused user kernels to die on start (making notebook execution impossible) for newly started user servers from about Mar 22 19:30 to Mar 23 09:45. Most users didn’t see any errors until start of class at about 9AM, since they were running servers that were previously started."
+    "text": "On February 28, 2018, a handful of users reported on piazza that there servers wouldn’t start. It was determined that all problematic servers were running on the same node. After the node was cordoned and rebooted, the student servers were able to start properly."
   },
   {
-    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#timeline",
-    "href": "incidents/2017-03-23-kernel-deaths-incident.html#timeline",
-    "title": "Weird upstream ipython bug kills kernels",
+    "objectID": "incidents/2018-02-28-hung-node.html#timeline",
+    "href": "incidents/2018-02-28-hung-node.html#timeline",
+    "title": "A node hangs, causing a subset of users to report issues",
     "section": "Timeline",
-    "text": "Timeline\n\nMarch 22, around 19:30\nA deployment is performed, finally deploying https://github.com/data-8/jupyterhub-k8s/pull/146 to production. It seemed to work fine on -dev, and on prod as well. However, the testing regimen was only to see if a notebook server would show up - not if a kernel would spawn.\n\n\nMar 23, 09:08\nStudents report that their kernels keep dying. This is confirmed to be a problem for all newly launched notebooks, in both prod and dev.\n\n\n09:16\nThe last change to the repo (an update of the single-user image) is reverted, to check if that was causing the problem. This does not improve the situation. Debugging continues, but with no obvious angles of attack.\n\n\n09:41\nAfter debugging produces no obvious culprits, the state of the entire infrastructure for prod is reverted to a known good state from a few days ago. This was done with:\n./deploy.py prod data8 25abea764121953538713134e8a08e0291813834\n25abea764121953538713134e8a08e0291813834 is the commit hash of a known good commit from March 19. Our disciplined adherence to immutable & reproducible deployment paid off, and we were able to restore new servers to working order with this!\nStudents are now able to resume working after a server restart. A mass restart is also performed to aid this.\nDev is left in a broken state in an attempt to debug.\n\n\n09:48\nA core Jupyter Notebook dev at BIDS attempts to debug the problem, since it seems to be with the notebook itself and not with JupyterHub.\n\n\n11:08\nCore Jupyter Notebook dev confirms that this makes no sense.\n\n\n14:55\nAttempts to isolate the bug start again, mostly by using git bisect to deploy different versions of our infrastructure to dev until we find what broke.\n\n\n15:30\nhttps://github.com/data-8/jupyterhub-k8s/pull/146 is identified as the culprit. It continues to not make sense.\n\n\n17:25\nA very involved and laborious revert of the offending part of the patch is done in https://github.com/jupyterhub/kubespawner/pull/37. Core Jupyter Notebook dev continues to confirm this makes no sense.\nhttps://github.com/data-8/jupyterhub-k8s/pull/152 is also merged, and deployed shortly after verifying that everything (including starting kernels & executing code) works fine on dev. Deployed to prod and everything is fine."
+    "text": "Timeline\n\n2018-02-28 21:21\nThree students report problems starting their server on piazza and a GSI links to the reports on slack. More reports come in by 21:27.\n\n\n21:30\nThe infrastructure team is alerted to the problem. The command kubectl --namespace=prod get pod -o wide | egrep -v -e prepull -e Running shows that all non-running pods were scheduled on the same node. Most of the pods have an “Unknown” status while the rest are in “Terminating”. The oldest problematic pod is 29m.\n\n\n21:34\nThe node k8s-pool1-19522833-9 is cordoned. It has a load of about 90 with no processes consuming much CPU. The node is rebooted via sysrq trigger. The hung pods remain stuck.\n\n\n21:39\nWhen the node comes back online, kubectl reports no more hung pods. Students are able to start their servers."
   },
   {
-    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#conclusion",
-    "href": "incidents/2017-03-23-kernel-deaths-incident.html#conclusion",
-    "title": "Weird upstream ipython bug kills kernels",
+    "objectID": "incidents/2018-02-28-hung-node.html#conclusion",
+    "href": "incidents/2018-02-28-hung-node.html#conclusion",
+    "title": "A node hangs, causing a subset of users to report issues",
     "section": "Conclusion",
-    "text": "Conclusion\nInsufficient testing procedures caused a new kind of outage (kernel dying) that we had not seen before. However, since our infrastructure was immutable & reproducible, our outage really only lasted about 40 minutes (from start of lab when students were starting containers until the revert). Deeper debugging produced a fix, but attempts to understand why the fix works are ongoing.\nUpdate: We have found and fixed the underlying issue"
+    "text": "Conclusion\nA problematic VM prevented nodes from launching pods. Once the VM was cordoned and rebooted, pods launch without trouble."
   },
   {
-    "objectID": "incidents/2017-03-23-kernel-deaths-incident.html#action-items",
-    "href": "incidents/2017-03-23-kernel-deaths-incident.html#action-items",
-    "title": "Weird upstream ipython bug kills kernels",
+    "objectID": "incidents/2018-02-28-hung-node.html#action-items",
+    "href": "incidents/2018-02-28-hung-node.html#action-items",
+    "title": "A node hangs, causing a subset of users to report issues",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nDocument and formalize the testing process for post-deployment checks.\nSet a short timeout (maybe ten minutes?) after which investigation temporarily stops and we revert our deployment to a known good state.\n\n\n\nUpstream KubeSpawner\n\nContinue investigating https://github.com/jupyterhub/kubespawner/issues/31, which was the core issue that prompted the changes that eventually led to the outage."
+    "text": "Action items\n\nProcess\n\nMonitor the cluster for non-running pods and send an alert if the count exceeds a threshold or if the non-running pods are clustered on the same node(s)."
   },
   {
-    "objectID": "incidents/2018-01-26-hub-slow-startup.html",
-    "href": "incidents/2018-01-26-hub-slow-startup.html",
-    "title": "Hub starts up very slow, causing outage for users",
+    "objectID": "incidents/2017-02-24-autoscaler-incident.html",
+    "href": "incidents/2017-02-24-autoscaler-incident.html",
+    "title": "Custom Autoscaler gonee haywire",
     "section": "",
-    "text": "On January 26, 2018, a new version of the helm chart was being installed on the production hub. Though the pod prepuller worked fine on the staging cluster, the prepuller never successfully finished on prod. This caused the CI to error because helm ran for too long. Additionally, the hub was taking a very long time to check user routes. After users were deleted in the hub’s orm and the hub was restarted, it came back up fairly quickly."
+    "text": "On the evening of February 24, 2017, a premature version of the Autoscaler script for the Datahub deployment was mistakenly run on the prod cluster, resulting in a large amount of nodes (roughly 30-40) being set as unschedulable for about 20 minutes. Though no information was lost nor service critically disturbed, it was necessary to manually re-enable these nodes to be scheduled."
   },
   {
-    "objectID": "incidents/2018-01-26-hub-slow-startup.html#summary",
-    "href": "incidents/2018-01-26-hub-slow-startup.html#summary",
-    "title": "Hub starts up very slow, causing outage for users",
+    "objectID": "incidents/2017-02-24-autoscaler-incident.html#summary",
+    "href": "incidents/2017-02-24-autoscaler-incident.html#summary",
+    "title": "Custom Autoscaler gonee haywire",
     "section": "",
-    "text": "On January 26, 2018, a new version of the helm chart was being installed on the production hub. Though the pod prepuller worked fine on the staging cluster, the prepuller never successfully finished on prod. This caused the CI to error because helm ran for too long. Additionally, the hub was taking a very long time to check user routes. After users were deleted in the hub’s orm and the hub was restarted, it came back up fairly quickly."
+    "text": "On the evening of February 24, 2017, a premature version of the Autoscaler script for the Datahub deployment was mistakenly run on the prod cluster, resulting in a large amount of nodes (roughly 30-40) being set as unschedulable for about 20 minutes. Though no information was lost nor service critically disturbed, it was necessary to manually re-enable these nodes to be scheduled."
   },
   {
-    "objectID": "incidents/2018-01-26-hub-slow-startup.html#timeline",
-    "href": "incidents/2018-01-26-hub-slow-startup.html#timeline",
-    "title": "Hub starts up very slow, causing outage for users",
+    "objectID": "incidents/2017-02-24-autoscaler-incident.html#timeline",
+    "href": "incidents/2017-02-24-autoscaler-incident.html#timeline",
+    "title": "Custom Autoscaler gonee haywire",
     "section": "Timeline",
-    "text": "Timeline\n\n2018-01-26 15:00\nThe helm chart for datahub was upgraded to a beta of v0.6 to make use of a new image puller. This was merged into the staging branch, successfully tested on the staging, and passed CI checks on prod. It was then merged to prod.\n\n\n15:15\nhelm times out because the prepuller never completes. It is determined that the master node on staging is cordoned while the master node on prod is not and has:\ntaints:\n  - effect: NoSchedule\n    key: node-role.kubernetes.io/master\n    timeAdded: null\n    value: \"true\"\nThe master is cordoned on prod and a new build is started in CI.\n\n\n15:33\nAfter CI times out again due to the prepuller, it is discovered that the master node has been uncordoned. Though the hub and proxy pods restart, the hub is taking a very long time to check user routes. It is slower than the most recent hub restart which was itself slow enough to warrant a new issue on jupyterhub, https://github.com/jupyterhub/jupyterhub/issues/1633.\n\n\n13:40\nIt is decided that the most expedient way to get the hub up is to delete users from the orm.\n\n\n13:50\nThe following command is run after the database is backed up:\ndelete from users where users.id in (select users.id from users join spawners on spawners.user_id = users.id where server_id is null);\ndeleting 4902 records. The hub pod is deleted and the hub comes up shortly after."
+    "text": "Timeline\nAs of this commit in the Autoscaler branch history, there exists a scale.py file that would based on the utilization of the cluster, mark a certain number of nodes unschedulable before attempting to shut down nodes with no pods in them. Unfortunately, this script was executed prematurely, and without configuration, looked to execute in whatever context currently specified in .kube/config, which ended up being the production cluster rather than the dev cluster.\n\n2017-02-24 11:14 PM\nScript is mistakenly executed. A bug in the calculations for the utilization of the cluster leads to about 40 nodes being marked as unschedulable. The mistake is noted immediately.\n\n\n2017-02-24 11:26 PM\nThe unschedulability of these nodes is reverted. All nodes in the cluster were first all set to be schedulable to ensure that no students current and future would be disturbed. Immediately after, 10 of the most idle nodes on the cluster were manually set to be unschedulable (to facilitate them later being manually descaled - to deal with https://github.com/data-8/infrastructure/issues/6) using kubectl cordon &lt;node_name&gt;."
   },
   {
-    "objectID": "incidents/2018-01-26-hub-slow-startup.html#conclusion",
-    "href": "incidents/2018-01-26-hub-slow-startup.html#conclusion",
-    "title": "Hub starts up very slow, causing outage for users",
+    "objectID": "incidents/2017-02-24-autoscaler-incident.html#conclusion",
+    "href": "incidents/2017-02-24-autoscaler-incident.html#conclusion",
+    "title": "Custom Autoscaler gonee haywire",
     "section": "Conclusion",
-    "text": "Conclusion\nAt 5000, the hub takes long enough to restart to inconvenience the number of active users at any one time."
+    "text": "Conclusion\nA cluster autoscaler script was accidentally run against the production cluster instead of the dev cluster, reducing capacity for new user logins for about 12 minutes. There was still enough capacity so we had no adverse effects."
   },
   {
-    "objectID": "incidents/2018-01-26-hub-slow-startup.html#action-items",
-    "href": "incidents/2018-01-26-hub-slow-startup.html#action-items",
-    "title": "Hub starts up very slow, causing outage for users",
-    "section": "Action items",
-    "text": "Action items\n\nProcess\n\nThe prepuller should be fixed so that helm does not time out.\nThe hub route checking should be parallelized so that startup is not slow.\nThe staging hub should be seeded with users so that scaling issues can be exposed prior to reaching production."
+    "objectID": "incidents/2017-02-24-autoscaler-incident.html#action-items",
+    "href": "incidents/2017-02-24-autoscaler-incident.html#action-items",
+    "title": "Custom Autoscaler gonee haywire",
+    "section": "Action Items",
+    "text": "Action Items\n\nDatahub Deployment Changes\n\nThe Autoscaler should not be run unless the context is explicitly set via environment variables or command line arguments. This is noted in the comments of the pull request for the Autoscaler.\nThe idea of the ‘current context’ should be abolished in all the tools we build / read.\n\n\n\nFuture organizational change\n\nUse a separate billing account for production vs development clusters. This makes it harder to accidentally run things on the wrong cluster"
   },
   {
-    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html",
-    "href": "incidents/2018-01-25-helm-chart-upgrade.html",
-    "title": "Accidental merge to prod brings things down",
+    "objectID": "incidents/2017-10-10-hung-nodes.html",
+    "href": "incidents/2017-10-10-hung-nodes.html",
+    "title": "Docker dies on a few Azure nodes",
     "section": "",
-    "text": "On January 25, 2018, a new version of the helm chart was installed on the staging hub. It was not immediately merged to production because there were active labs throughout the day. While preparing another course’s hub via Travis CI, the Data 8 change was accidentally merged from staging to production. This production hub went down because the new helm chart’s jupyterhub image was broken."
+    "text": "On Oct 10, 2017, some user pods were not starting or terminating correctly. After checking node status, it was found that all affected pods were running on two specific nodes. The docker daemon wasn’t responsive on these nodes so they were cordoned off. User pods were then able to start correctly."
   },
   {
-    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#summary",
-    "href": "incidents/2018-01-25-helm-chart-upgrade.html#summary",
-    "title": "Accidental merge to prod brings things down",
+    "objectID": "incidents/2017-10-10-hung-nodes.html#summary",
+    "href": "incidents/2017-10-10-hung-nodes.html#summary",
+    "title": "Docker dies on a few Azure nodes",
     "section": "",
-    "text": "On January 25, 2018, a new version of the helm chart was installed on the staging hub. It was not immediately merged to production because there were active labs throughout the day. While preparing another course’s hub via Travis CI, the Data 8 change was accidentally merged from staging to production. This production hub went down because the new helm chart’s jupyterhub image was broken."
+    "text": "On Oct 10, 2017, some user pods were not starting or terminating correctly. After checking node status, it was found that all affected pods were running on two specific nodes. The docker daemon wasn’t responsive on these nodes so they were cordoned off. User pods were then able to start correctly."
   },
   {
-    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#timeline",
-    "href": "incidents/2018-01-25-helm-chart-upgrade.html#timeline",
-    "title": "Accidental merge to prod brings things down",
+    "objectID": "incidents/2017-10-10-hung-nodes.html#timeline",
+    "href": "incidents/2017-10-10-hung-nodes.html#timeline",
+    "title": "Docker dies on a few Azure nodes",
     "section": "Timeline",
-    "text": "Timeline\n\n2018-01-25 14:30\nThe helm chart for datahub was upgraded to a beta of v0.6 to make use of a new image puller. This was merged into the staging branch. After some initial debugging, the helm chart was installed successfully and the image puller worked correctly. However, the staging hub was not tested.\nSince labs were scheduled throughout the day until 7p, it was decided to delay the upgrade of the production hub until after 7p.\n\n\n15:30\nWhile a different hub was being managed in Travis CI, the production hub for Data 8 was accidentally upgraded. This upgrade brought with it the faulty hub image from staging which wasn’t working.\n\n\n16:11\nGSIs report in slack that the hub is down for lab users. It is confirmed that the hub process has crashed due to a shared C library included from a python library. It is decided that the quickest way to bring the hub back up is to downgrade the helm-chart back to v0.5.0.\n\n\n16:35\nThe chart is installed into the staging repo, merged to staging, and checked on the staging hub. It is then merged into production and brought online there."
+    "text": "Timeline\n\n2017-05-09 10:45a\nA report in the course Piazza said that two students couldn’t start their servers. The /hub/admin interface was not able to start them either. It was reported that the students may have run out of memory.\n\n\n12:29p\nThe user pods were stuck in Terminating state and would not respond to explicit delete. The pods were forcefully deleted with kubectl --namespace=prod delete pod jupyter-&lt;name&gt; --grace-period=0 --force. The user pods started correctly via /hub/admin.\n\n\n13:27\nIt was reported in the course slack that another student’s server wasn’t starting correctly. After checking one of the pod logs, it was observed that the node hosting the pods, k8s-pool1-19522833-13, was also hosting many more pods stuck in a Terminating state. docker ps was hanging on that node. The node was cordoned.\n\n\n13:42\nIt was reported in slack that the student’s server was able to start.\nBy this time, the cluster was checked for all pods to see if any other nodes were hosting an unusual number of pods in Terminating. It was found that k8s-pool2-19522833-9 was in a similar state. All stuck pods on that node were forcefully deleted and the node was also cordoned. docker ps was hung on that node too. pool2-…-9 had a load of 530 while pool1-…-13 had a load of 476. On the latter, hypercube was at 766% cpu utilization while it was nominal on the former. Node pool1-…-13 was rebooted from the shell however it did not come back online. The node was manually restarted from the Azure portal but it still didn’t come back.\nA node previously cordoned on another day, pool1-…-14, was rebooted. It came back online and was uncordoned.\n\n\n13:51\nSome relevant systemctl status docker logs were captured from pool2-…-9:\nOct 10 20:55:30 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:30.790401257Z\" level=error msg=\"containerd: start container\" error=\"containerd: container did not start before the specified timeout\" id=abd267ef08b4a4184e19307be784d62470f9a713b59e406249c6cdf0bb333260\nOct 10 20:55:30 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:30.790923460Z\" level=error msg=\"Create container failed with error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:30 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:30.810309575Z\" level=error msg=\"Handler for POST /v1.24/containers/abd267ef08b4a4184e19307be784d62470f9a713b59e406249c6cdf0bb333260/start returned error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.146453953Z\" level=error msg=\"containerd: start container\" error=\"containerd: container did not start before the specified timeout\" id=2ba6787503ab6123b509811fa44c7e42986de0b800cc4226e2ab9484f54e8741\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.147565759Z\" level=error msg=\"Create container failed with error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.166295370Z\" level=error msg=\"Handler for POST /v1.24/containers/2ba6787503ab6123b509811fa44c7e42986de0b800cc4226e2ab9484f54e8741/start returned error: containerd: container did not start before the specified timeout\"\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.169360588Z\" level=error msg=\"Handler for GET /v1.24/containers/json returned error: write unix /var/run/docker.sock-&gt;@: write: broken pipe\"\nOct 10 20:55:36 k8s-pool2-19522833-9 dockerd[1237]: http: multiple response.WriteHeader calls\nOct 10 20:55:36 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:36.280209444Z\" level=error msg=\"Handler for GET /v1.24/containers/610451d9d86a58117830ea7c0189f6157ba9a9602739ee23723e923de8c7e23e/json returned error: No such container: 610451d9d86a58117830ea7c0189f6157ba9a9602739ee23723e923de8c7e23e\"\nOct 10 20:55:39 k8s-pool2-19522833-9 docker[1237]: time=\"2017-10-10T20:55:39.095888009Z\" level=error msg=\"Handler for GET /v1.24/containers/54b64ca3c1e7ef4a04192ccdaf1cb9309d73acebd7a08e13301f3263de3d376a/json returned error: No such container: 54b64ca3c1e7ef4a04192ccdaf1cb9309d73acebd7a08e13301f3263de3d376a\"\n\n\n14:00\ndatahub@k8s-pool2-19522833-9:~$ ps aux | grep exe | wc -l\n520\ndatahub@k8s-pool2-19522833-9:~$ ps aux | grep exe | head -5\nroot        329  0.0  0.0 126772  9812 ?        Dsl  00:36   0:00 /proc/self/exe init\nroot        405  0.0  0.0  61492  8036 ?        Dsl  00:36   0:00 /proc/self/exe init\nroot        530  0.0  0.0 127028  8120 ?        Dsl  00:36   0:00 /proc/self/exe init\nroot        647  0.0  0.0 127028  8124 ?        Dsl  13:07   0:00 /proc/self/exe init\nroot        973  0.0  0.0  77884  8036 ?        Dsl  13:10   0:00 /proc/self/exe init\n\n\n14:30\npool1-…-13 was manually stopped in the Azure portal, then manually started. It came back online afterwards and docker was responsive. It was uncordoned.\npool2-…-9 was manually stopped in the Azure portal.\n\n\n14:45\npool2-…-9 completed stopping and was manually started in the Azure portal.\n\n\n17:25\nIt was observed that /var/lib/docker on pool1-19522833-13/10.240.0.7 was on / (sda) and not on /mnt (sdb)."
   },
   {
-    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#conclusion",
-    "href": "incidents/2018-01-25-helm-chart-upgrade.html#conclusion",
-    "title": "Accidental merge to prod brings things down",
+    "objectID": "incidents/2017-10-10-hung-nodes.html#conclusion",
+    "href": "incidents/2017-10-10-hung-nodes.html#conclusion",
+    "title": "Docker dies on a few Azure nodes",
     "section": "Conclusion",
-    "text": "Conclusion\nA relatively large change was made to the hub configuration with insufficient testing on the staging server. This was compounded when the change was accidentally merged to production."
+    "text": "Conclusion\nDocker was hung on two nodes, preventing pods from starting or stopping correctly."
   },
   {
-    "objectID": "incidents/2018-01-25-helm-chart-upgrade.html#action-items",
-    "href": "incidents/2018-01-25-helm-chart-upgrade.html#action-items",
-    "title": "Accidental merge to prod brings things down",
+    "objectID": "incidents/2017-10-10-hung-nodes.html#action-items",
+    "href": "incidents/2017-10-10-hung-nodes.html#action-items",
+    "title": "Docker dies on a few Azure nodes",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nAdmins should refamiliarize themselves with the deployment policy to check the staging hub before changes are merged to production.\nDetermine if there is a way to block merges to production if the staging hub is not online.\nDetermine if there is a way to contextualize the Travis CI interface so that it is obvious which deployment is being managed."
+    "text": "Action items\n\nProcess\n\nWhen there are multiple reports of student servers not starting or stopping correctly, check to see if the user pods were run on the same node(s).\nDetermine how many nodes are not mounting /var/lib/docker on sdb1.\n\n\n\nMonitoring\n\nLook for elevated counts of pods stuck in Terminating state. For example, kubectl --namespace=prod get pod -o wide| grep Terminating"
   },
   {
-    "objectID": "incidents/2019-05-01-service-account-leak.html",
-    "href": "incidents/2019-05-01-service-account-leak.html",
-    "title": "Service Account key leak incident",
+    "objectID": "incidents/2017-03-20-too-many-volumes.html",
+    "href": "incidents/2017-03-20-too-many-volumes.html",
+    "title": "Too many volumes per disk leave students stuck",
     "section": "",
-    "text": "Service account keys that granted restricted access to some of our cloud services were inadvertently leaked on GitHub. Google immediately notified us in seconds, and the credentials were revoked within the next few minutes."
+    "text": "From sometime early March 20 2017 till about 1300, some new student servers were stuck in Pending forever, giving them 500 errors. This was an unintended side-effect of reducing student memory limit to 1G while keeping the size of our nodes constant, causing us to hit a Google Cloud limit on number of disks per node. This was fixed by spawning more nodes that were smaller."
   },
   {
-    "objectID": "incidents/2019-05-01-service-account-leak.html#summary",
-    "href": "incidents/2019-05-01-service-account-leak.html#summary",
-    "title": "Service Account key leak incident",
+    "objectID": "incidents/2017-03-20-too-many-volumes.html#summary",
+    "href": "incidents/2017-03-20-too-many-volumes.html#summary",
+    "title": "Too many volumes per disk leave students stuck",
     "section": "",
-    "text": "Service account keys that granted restricted access to some of our cloud services were inadvertently leaked on GitHub. Google immediately notified us in seconds, and the credentials were revoked within the next few minutes."
+    "text": "From sometime early March 20 2017 till about 1300, some new student servers were stuck in Pending forever, giving them 500 errors. This was an unintended side-effect of reducing student memory limit to 1G while keeping the size of our nodes constant, causing us to hit a Google Cloud limit on number of disks per node. This was fixed by spawning more nodes that were smaller."
   },
   {
-    "objectID": "incidents/2019-05-01-service-account-leak.html#impact",
-    "href": "incidents/2019-05-01-service-account-leak.html#impact",
-    "title": "Service Account key leak incident",
-    "section": "Impact",
-    "text": "Impact\nDeployments are paused until this was fixed."
+    "objectID": "incidents/2017-03-20-too-many-volumes.html#timeline",
+    "href": "incidents/2017-03-20-too-many-volumes.html#timeline",
+    "title": "Too many volumes per disk leave students stuck",
+    "section": "Timeline",
+    "text": "Timeline\n\nMarch 18, 16:30\nRAM per student is reduced from 2G to 1G, as a resource optimization measure. The size of our nodes remains the same (26G RAM), and many are cordonned off and slowly decommissioned over the coming few days.\nLife seems fine, given the circumstances.\n\n\nMarch 20, 12:44\nNew student servers report a 500 error preventing them from logging on. This is deemed widespread & not an isolated incident.\n\n\n12:53\nA kubectl describe pod on an affected student’s pod shows it’s stuck in Pending state, with an error message:\npod failed to fit in any node fit failure on node (XX): MaxVolumeCount\nThis seems to be common problem for all the new student servers, which are all stuck in Pending state.\nGoogling leads to https://github.com/kubernetes/kubernetes/issues/24317 - even though Google Compute Engine can handle more than 16 disks per node (we had checked this before deploying), Kubernetes itself still can not. This wasn’t foreseen, and seemed to be the direct cause of the incident.\n\n\n13:03\nA copy of the instance template that is used by Google Container Engine is made and then modified to spawn smaller nodes (n1-highmem-2 rather than n1-highmem-4). The managed instance group used by Google Container Engine is then modified to use the new template. This was the easiest way to not distrupt students for whom things are working, while also allowing new students to be able to log in.\nThis new instance group was then set to expand for 30 new nodes, which will provide capacity for about 12 students each. populate.bash was also run to make sure that students pods start up on time in the newnodes.\n\n\n13:04\nThe simple autoscaler is stopped, on fear that it’ll be confused by the unusual mixed state of the nodes and do something wonky.\n\n\n13:11\nAll the new nodes are online, and populate.bash has completed. Pods start leaving the Pending state.\nHowever, since it’s been more than the specified timeout that JupyterHub will wait before giving up on Pod (5 minutes), JupyterHub doesn’t know the pods exist. This causes state of cluster + state in JupyterHub to go out of sync, causing the dreaded ‘redirected too many times’ error. Admins need to manually stop and start user pods in the control panel as users report this to fix this issue.\n\n\n14:23\nThe hub and proxy pods are restarted since there were plenty of ‘redirected too many times’ errors. This seems to catch most users state, although some requests still failed with a 599 timeout (similar to an earlier incident, but much less frequent). A long tail of manual user restarts are performed by admins over the next few days."
   },
   {
-    "objectID": "incidents/2019-05-01-service-account-leak.html#timeline",
-    "href": "incidents/2019-05-01-service-account-leak.html#timeline",
-    "title": "Service Account key leak incident",
-    "section": "Timeline",
-    "text": "Timeline\n\nMay 1 2019, 3:18 PM\nA template + documentation for creating new hubs easily is pushed to GitHub as a pull request. This inadvertently contained live credentials for pushing & pulling our (already public) docker images, and for access to our kubernetes clusters.\nGoogle immediately notified us via email within seconds that this might be a breach.\n\n\n3:19 PM\nDiscussion and notification starts in slack about dealing with the issue.\n\n\n3:27 PM\nBoth keys are revoked so they are no longer valid credentials.\n\n\n3:36 PM\nAll in-use resources are checked, and verified to not be compromised by automated bots looking for leaked accounts.\n\n\n3:40 PM\nAn email is sent out to all owners of the compromised project (ucb-datahub-2018) giving an all-clear."
+    "objectID": "incidents/2017-03-20-too-many-volumes.html#action-items",
+    "href": "incidents/2017-03-20-too-many-volumes.html#action-items",
+    "title": "Too many volumes per disk leave students stuck",
+    "section": "Action Items",
+    "text": "Action Items\n\nUpstream: Kubernetes\n\nKeep an eye on the status of the bug we ran into\n\n\n\nUpstream: JupyterHub\n\nTrack down and fix the ‘too many redirects’ issue at source. Issue\n\n\n\nCleanup\n\nDelete all the older larger nodes that are no longer in use. (Done!)\n\n\n\nMonitoring\n\nHave alerting for when there are any number of pods in Pending state for a non-negligible amount of time. There is always something wrong when this happens."
   },
   {
-    "objectID": "incidents/2019-05-01-service-account-leak.html#action-items",
-    "href": "incidents/2019-05-01-service-account-leak.html#action-items",
-    "title": "Service Account key leak incident",
-    "section": "Action items",
-    "text": "Action items\n\nDon’t duplicate service key credentials across multiple hubs. Issue\nSwitch to a different secret management strategy than what we have now. Issue"
+    "objectID": "incidents/index.html",
+    "href": "incidents/index.html",
+    "title": "Incident Reports",
+    "section": "",
+    "text": "Order By\n       Default\n         \n          Date - Oldest\n        \n         \n          Date - Newest\n        \n         \n          Title\n        \n     \n  \n    \n      \n      \n    \n\n\n\n\n\nDate\n\n\nTitle\n\n\n\n\n\n\nFeb 9, 2017\n\n\nJupyterHub db manual overwrite\n\n\n\n\nFeb 24, 2017\n\n\nCustom Autoscaler gonee haywire\n\n\n\n\nFeb 24, 2017\n\n\nProxy eviction strands user\n\n\n\n\nMar 6, 2017\n\n\nNon-matching hub image tags cause downtime\n\n\n\n\nMar 20, 2017\n\n\nToo many volumes per disk leave students stuck\n\n\n\n\nMar 23, 2017\n\n\nWeird upstream ipython bug kills kernels\n\n\n\n\nApr 3, 2017\n\n\nCustom autoscaler does not scale up when it should\n\n\n\n\nMay 9, 2017\n\n\nOops we forgot to pay the bill\n\n\n\n\nOct 10, 2017\n\n\nDocker dies on a few Azure nodes\n\n\n\n\nOct 19, 2017\n\n\nBilling confusion with Azure portal causes summer hub to be lost\n\n\n\n\nJan 25, 2018\n\n\nAccidental merge to prod brings things down\n\n\n\n\nJan 26, 2018\n\n\nHub starts up very slow, causing outage for users\n\n\n\n\nFeb 6, 2018\n\n\nAzure PD refuses to detach, causing downtime for data100\n\n\n\n\nFeb 28, 2018\n\n\nA node hangs, causing a subset of users to report issues\n\n\n\n\nJun 11, 2018\n\n\nAzure billing issue causes downtime\n\n\n\n\nFeb 25, 2019\n\n\nAzure Kubernetes API Server outage causes downtime\n\n\n\n\nMay 1, 2019\n\n\nService Account key leak incident\n\n\n\n\nJan 20, 2022\n\n\nHubs throwing 505 errors\n\n\n\n\nFeb 1, 2024\n\n\nCore nodes being autoscaled, configurable HTTP proxy crashes\n\n\n\n\n\nNo matching items",
+    "crumbs": [
+      "Contributing",
+      "Incident Reports"
+    ]
   },
   {
-    "objectID": "incidents/2017-04-03-cluster-full-incident.html",
-    "href": "incidents/2017-04-03-cluster-full-incident.html",
-    "title": "Custom autoscaler does not scale up when it should",
+    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html",
+    "href": "incidents/2017-03-06-helm-config-image-mismatch.html",
+    "title": "Non-matching hub image tags cause downtime",
     "section": "",
-    "text": "On April 3, 2017, as students were returning from spring break, the cluster wasn’t big enough in time and several students had errors spawning. This was because the simple-autoscaler was ‘stuck’ on a populate call. More capacity was manually added, the pending pods were deleted & this seemed to fix the outage."
+    "text": "On the evening of Mar 6, the hub on prod would not come up after an upgrade. The upgrade was to accommodate a new disk for cogneuro that had been tested on dev. After some investigation it was determined that the helm’s config did not match the hub’s image. After the hub image was rebuilt and pushed out, then tested on dev, it was pushed out to prod. The problem was fixed in about 40 minutes.\nA few days later (March 12), similar almost outage is avoided when -dev breaks and deployment is put on hold. More debugging shows the underlying cause is that git submodules are hard to use. More documentation is provided, and downtime is averted!"
   },
   {
-    "objectID": "incidents/2017-04-03-cluster-full-incident.html#summary",
-    "href": "incidents/2017-04-03-cluster-full-incident.html#summary",
-    "title": "Custom autoscaler does not scale up when it should",
+    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#summary",
+    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#summary",
+    "title": "Non-matching hub image tags cause downtime",
     "section": "",
-    "text": "On April 3, 2017, as students were returning from spring break, the cluster wasn’t big enough in time and several students had errors spawning. This was because the simple-autoscaler was ‘stuck’ on a populate call. More capacity was manually added, the pending pods were deleted & this seemed to fix the outage."
+    "text": "On the evening of Mar 6, the hub on prod would not come up after an upgrade. The upgrade was to accommodate a new disk for cogneuro that had been tested on dev. After some investigation it was determined that the helm’s config did not match the hub’s image. After the hub image was rebuilt and pushed out, then tested on dev, it was pushed out to prod. The problem was fixed in about 40 minutes.\nA few days later (March 12), similar almost outage is avoided when -dev breaks and deployment is put on hold. More debugging shows the underlying cause is that git submodules are hard to use. More documentation is provided, and downtime is averted!"
   },
   {
-    "objectID": "incidents/2017-04-03-cluster-full-incident.html#timeline",
-    "href": "incidents/2017-04-03-cluster-full-incident.html#timeline",
-    "title": "Custom autoscaler does not scale up when it should",
+    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#timeline",
+    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#timeline",
+    "title": "Non-matching hub image tags cause downtime",
     "section": "Timeline",
-    "text": "Timeline\n\nOver spring break week\nThe cluster is scaled down to a much smaller size (7 machines), and the simple scaler is left running.\n\n\n2017-04-03 11:32\nStudents report datahub isn’t working on Piazza, and lots of Pods in PENDING state.\nDoing a kubectl --namespace=datahub describe pod &lt;pod-name&gt; said the pod was unschedulable because there wasn’t enough RAM in the cluster. This clearly implied the cluster wasn’t big enough.\nLooking at the simple scaler shows it was ‘stuck’ at a populate.bash call, and wasn’t scaling up fast enough.\n\n\n11:35\nThe cluster is manually scaled up to 30 nodes:\ngcloud compute instance-groups managed resize gke-prod-highmem-pool-0df1a536-grp --size=30\nAt the same time, pods stuck in Pending state are deleted so they don’t become ghost pods, with:\nkubectl --namespace=datahub get pod | grep -v Running | grep -P 'm$' | awk '{print $1;}' | xargs -L1 kubectl --namespace=datahub delete pod\n\n\n11:40\nThe nodes have come up, so a populate.bash call is performed to pre-populate all user container images on the new nodes.\nUsers in Pending state are deleted again.\n\n\n11:46\nThe populate.bash call is complete, and everything is back online!"
+    "text": "Timeline\nAll times in PST\n\nMarch 6 2017 22:59\ndev changes are deployed but hub does not start correctly. The describe output for the hub shows repeated instances of:\nError syncing pod, skipping: failed to “StartContainer” for “hub-container” with CrashLoopBackOff: “Back-off 10s restarting failed container=hub-container pod=hub-deployment-3498421336-91gp3_datahub-dev(bfe7d8bd-0303-11e7-ade6-42010a80001a)\nhelm chart for -dev is deleted and reinstalled.\n\n\n23:11\ndev changes are deployed successfully and tested. cogneuro’s latest data is available.\n\n\n23:21\nChanges are deployed to prod. The hub does not start properly. get pod -o=yaml on the hub pod shows that the hub container has terminated. The hub log shows that it failed due to a bad configuration parameter.\n\n\n21:31\nWhile the helm chart had been updated from git recently, the latest tag for the hub did not correspond with the one in either prod.yaml or dev.yaml.\n\n\n21:41\nThe hub image is rebuilt and pushed out.\n\n\n21:45\nThe hub is deployed on -dev.\n\n\n21:46\nThe hub is tested on -dev then deployed on -prod.\n\n\n21:50\nThe hub is tested on -prod. Students are reporting that the hub had been down.\n\n\nMarch 12 19:57\nA new deploy is attempted on -dev, but runs into same error. Deployments are halted for more debugging this time, and more people are called on.\n\n\n23:21\nMore debugging reveals that the commit update looked like this:\ndiff --git a/chart b/chart\nindex e38aba2..c590340 160000\n--- a/chart\n+++ b/chart\n@@ -1 +1 @@\n-Subproject commit e38aba2c5601de30c01c6f3c5cad61a4bf0a1778\n+Subproject commit c59034032f8870d16daba7599407db7e6eb53e04\ndiff --git a/data8/dev.yaml b/data8/dev.yaml\nindex 2bda156..ee5987b 100644\n--- a/data8/dev.yaml\n+++ b/data8/dev.yaml\n@@ -13,7 +13,7 @@ publicIP: \"104.197.166.226\"\n\n singleuser:\n   image:\n-    tag: \"e4af695\"\n+    tag: \"1a6c6d8\"\n   mounts:\n     shared:\n       cogneuro88: \"cogneuro88-20170307-063643\"\nOnly the tag should’ve been the only thing updated. The chart submodule is updated to c59034032f8870d16daba7599407db7e6eb53e04, which is from February 25 (almost two weeks old). This is the cause of the hub failing, since it is using a really old chart commit with a new hub image."
   },
   {
-    "objectID": "incidents/2017-04-03-cluster-full-incident.html#conclusion",
-    "href": "incidents/2017-04-03-cluster-full-incident.html#conclusion",
-    "title": "Custom autoscaler does not scale up when it should",
+    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#section-8",
+    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#section-8",
+    "title": "Non-matching hub image tags cause downtime",
+    "section": "23:27",
+    "text": "23:27\nIt is determined that incomplete documentation about deployment processes caused git submodule update to be not run after a git pull, and so the chart was being accidentally moved back to older commits. Looking at the commit that caused the outage on March 6 showed the exact same root cause."
+  },
+  {
+    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#conclusion",
+    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#conclusion",
+    "title": "Non-matching hub image tags cause downtime",
     "section": "Conclusion",
-    "text": "Conclusion\nOur simple scaler didn’t scale up fast enough when a large number of students came back online quickly after a time of quiet (spring break). Took a while for this to get noticed, and manual scaling fixed everything."
+    "text": "Conclusion\nGit submodules are hard to use, and break most people’s mental model of how git works. Since our deployment requires that the submodule by in sync with the images used, this caused an outage."
   },
   {
-    "objectID": "incidents/2017-04-03-cluster-full-incident.html#action-items",
-    "href": "incidents/2017-04-03-cluster-full-incident.html#action-items",
-    "title": "Custom autoscaler does not scale up when it should",
+    "objectID": "incidents/2017-03-06-helm-config-image-mismatch.html#action-items",
+    "href": "incidents/2017-03-06-helm-config-image-mismatch.html#action-items",
+    "title": "Non-matching hub image tags cause downtime",
     "section": "Action items",
-    "text": "Action items\n\nProcess\n\nWhen coming back from breaks, pre-scale the cluster back up.\nConsider cancelling spring break.\n\n\n\nMonitoring\n\nHave monitoring for pods stuck in non-Running states"
+    "text": "Action items\n\nProcess\n\nMake sure we treat any errors in -dev exactly like we would in prod. Any deployment error in prod should immediately halt future deployments & require a rollback or resolution before proceeding.\nWrite down actual deployment documentation & a checklist.\nMove away from git submodules to a separate versioned chart repository."
   },
   {
-    "objectID": "users/authentication.html",
-    "href": "users/authentication.html",
-    "title": "User Authentication",
+    "objectID": "incidents/2017-02-24-proxy-death-incident.html",
+    "href": "incidents/2017-02-24-proxy-death-incident.html",
+    "title": "Proxy eviction strands user",
     "section": "",
-    "text": "Most of our hubs use a Canvas instance, bcourses.berkeley.edu, for authentication. The hubs not using CanvasOAuthenticator are edx (LTI1Authenticator), highschool (GoogleOAuthenticator), and workshop (dummy).",
-    "crumbs": [
-      "Contributing",
-      "Using DataHub",
-      "User Authentication"
-    ]
+    "text": "On the evening of Feb 23, several students started experiencing 500 errors in trying to access datahub. The proxy had died because of a known issue, and it took a while for the hub to re-add all the user routes to the proxy. Some students’ needed their servers to be manually restarted, due to a JupyterHub spawner bug that is showing up at scale. Everything was fixed in about 40 minutes."
   },
   {
-    "objectID": "users/authentication.html#authorization",
-    "href": "users/authentication.html#authorization",
-    "title": "User Authentication",
-    "section": "Authorization",
-    "text": "Authorization\nAnyone who can log in to bCourses can log into our Canvas-based hubs. This includes all Berkeley affiliates. If you have a working berkeley.edu email account, you can most likely log in to bCourses, and hence to our JupyterHubs.\nStudents have access for 9 months after they graduate. If they have an incomplete, they have 13 months of access instead.",
-    "crumbs": [
-      "Contributing",
-      "Using DataHub",
-      "User Authentication"
-    ]
+    "objectID": "incidents/2017-02-24-proxy-death-incident.html#summary",
+    "href": "incidents/2017-02-24-proxy-death-incident.html#summary",
+    "title": "Proxy eviction strands user",
+    "section": "",
+    "text": "On the evening of Feb 23, several students started experiencing 500 errors in trying to access datahub. The proxy had died because of a known issue, and it took a while for the hub to re-add all the user routes to the proxy. Some students’ needed their servers to be manually restarted, due to a JupyterHub spawner bug that is showing up at scale. Everything was fixed in about 40 minutes."
   },
   {
-    "objectID": "users/authentication.html#non-berkeley-affiliates",
-    "href": "users/authentication.html#non-berkeley-affiliates",
-    "title": "User Authentication",
-    "section": "Non-Berkeley affiliates",
-    "text": "Non-Berkeley affiliates\nIf someone who doesn’t have a berkeley.edu account wants to use the JupyterHubs, they need to obtain a CalNet Sponsored Guest account and get added to a bCourses course.",
-    "crumbs": [
-      "Contributing",
-      "Using DataHub",
-      "User Authentication"
-    ]
+    "objectID": "incidents/2017-02-24-proxy-death-incident.html#timeline",
+    "href": "incidents/2017-02-24-proxy-death-incident.html#timeline",
+    "title": "Proxy eviction strands user",
+    "section": "Timeline",
+    "text": "Timeline\nAll times in PST\n\n21:10:57\nThe proxy pod is evicted, due to a known issue that is currently being worked on. Users start running into issue now, with connection failures.\n\n\n21:11:04\nNew proxy pod is started by kubernetes, and starts accepting connections. However, the JupyterHub model currently has the proxy starting with no state about user routes, and so the users’ requests aren’t being routed to their notebook pods. This manifests as errors for users.\nThe hub process is supposed to poll the proxy every 300s, and repopulate the route table when it notices it is empty. The hub does this at some point in the next 300s (we do not know when), and starts repopulating the route table. As routes get added for currently users, their notebook starts working again.\n\n\n21:11:52\nThe repopulate process starts running into issues - it is making far too many http requests (to the kubernetes and proxy APIs) that it starts running into client side limits on tornado http client (which is what we use to make these requests). This causes them to time out on the request queue. We were running into https://github.com/tornadoweb/tornado/issues/1400. Not all requests fail - for those that succeed, the students are able to access their notebooks.\nThe repopulate process takes a while to process, and errors for a lot of students who are left with notebook in inconsistent state - JupyterHub thinks their notebook is running but it isn’t, or vice versa. Lots of 500s for users.\n\n\n21:14\nReports of errors start reaching the Slack channel + Piazza.\nThe repopulate process keeps being retried, and notebooks for users slowly come back. Some users are ‘stuck’ in a bad state, however - their notebook isn’t running, but JupyterHub thinks it is (or vice versa).\n\n\n21:34\nMost users are fine by now. For those still with problems, a forced delete from the admin interface + a start works, since this forces JupyterHub to really check if they’re there or not.\n\n\n22:03\nLast reported user with 500 error is fixed, and datahub is fully operational again."
   },
   {
-    "objectID": "users/authentication.html#troubleshooting",
-    "href": "users/authentication.html#troubleshooting",
-    "title": "User Authentication",
-    "section": "Troubleshooting",
-    "text": "Troubleshooting\nIf you can log in to bCourses but not to any of the JupyterHubs, please contact us.\nIf you can not log in to bCourses, please contact bCourses support.",
-    "crumbs": [
-      "Contributing",
-      "Using DataHub",
-      "User Authentication"
-    ]
+    "objectID": "incidents/2017-02-24-proxy-death-incident.html#conclusion",
+    "href": "incidents/2017-02-24-proxy-death-incident.html#conclusion",
+    "title": "Proxy eviction strands user",
+    "section": "Conclusion",
+    "text": "Conclusion\nThis is almost a ‘perfect storm’ event. Three things colluded to make this outage happen:\n\nThe inodes issue, which causes containers to fail randomly\nThe fact that the proxy is a single point of failure with a longish recovery time in current JupyterHub architecture.\nKubeSpawner’s current design is inefficient at very high user volumes, and its request timeouts & other performance characteristics had not been tuned (because we have not needed to before).\n\nWe have both long term (~1-2 months) architectural fixes as well as short term tuning in place for all three of these issues."
+  },
+  {
+    "objectID": "incidents/2017-02-24-proxy-death-incident.html#action-items",
+    "href": "incidents/2017-02-24-proxy-death-incident.html#action-items",
+    "title": "Proxy eviction strands user",
+    "section": "Action items",
+    "text": "Action items\n\nUpstream JupyterHub\n\nWork on abstracting the proxy interface, so the proxy is no longer a single point of failure. Issue\n\n\n\nUpstream KubeSpawner\n\nRe-architect the spawner to make a much smaller number of HTTP requests. DataHub has become big enough that this is a problem. Issue\nTune the HTTP client kubespawner uses. This would be an interim solution until (1) gets fixed. Issue\n\n\n\nDataHub configuration\n\nSet resource requests explicitly for hub and proxy, so they have less chance of getting evicted. Issue\nReduce the interval at which the hub checks to see if the proxy is running. PR\nSpeed up the fix for the inodes issue which is what triggered this whole issue."
   },
   {
     "objectID": "users/hubs.html",
diff --git a/sitemap.xml b/sitemap.xml
index 5f6189c0b..c50134f67 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,262 +2,258 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/cluster-config.html</loc>
-    <lastmod>2024-10-24T05:10:59.355Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/index.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/transition-image.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/github-token.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/cheatsheet.html</loc>
-    <lastmod>2024-10-24T05:10:59.355Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/clusterswitch.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/rebuild-postgres-image.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/calendar-scaler.html</loc>
-    <lastmod>2024-10-24T05:10:59.355Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/delete-hub.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/core-pool.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/managing-multiple-user-image-repos.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/new-hub.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/credentials.html</loc>
-    <lastmod>2024-10-24T05:10:59.355Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/policy/create_policy.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/policy/index.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/policy/principles.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/index.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/datahub.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/stat159.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/edx.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/stat20.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/data102.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/private-repo.html</loc>
-    <lastmod>2024-10-24T05:10:59.367Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.839Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-24-proxy-death-incident.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-04-03-cluster-full-incident.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-03-06-helm-config-image-mismatch.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2019-05-01-service-account-leak.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/index.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2018-01-25-helm-chart-upgrade.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-03-20-too-many-volumes.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2018-01-26-hub-slow-startup.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-10-10-hung-nodes.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-03-23-kernel-deaths-incident.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-24-autoscaler-incident.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-09-datahub-db-outage.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2018-02-28-hung-node.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2024-core-node-incidents.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2018-02-06-hub-db-dir.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2018-06-11-course-subscription-canceled.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-05-09-gce-billing.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-10-19-course-subscription-canceled.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2022-01-20-package-dependency-upgrade-incident.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/incidents/2019-02-25-k8s-api-server-down.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
-  </url>
-  <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-09-datahub-db-outage-pvc-recreate-script.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-10-19-course-subscription-canceled.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2022-01-20-package-dependency-upgrade-incident.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2018-06-11-course-subscription-canceled.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-05-09-gce-billing.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2024-core-node-incidents.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2018-02-06-hub-db-dir.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-09-datahub-db-outage.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2018-02-28-hung-node.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-03-23-kernel-deaths-incident.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-24-autoscaler-incident.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2018-01-26-hub-slow-startup.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-10-10-hung-nodes.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2018-01-25-helm-chart-upgrade.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-03-20-too-many-volumes.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2019-05-01-service-account-leak.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/index.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/incidents/2017-04-03-cluster-full-incident.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-03-06-helm-config-image-mismatch.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
-    <loc>https://docs.datahub.berkeley.edu/users/authentication.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <loc>https://docs.datahub.berkeley.edu/incidents/2017-02-24-proxy-death-incident.html</loc>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/data100.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/r.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/prob140.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/users/hubs/shiny.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/policy/policy_deploy_mainhubs.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/policy/policy_create_hubs.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/policy/storage-retention.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/storage.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/pre-reqs.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/course-config.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/new-image.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/rebuild-hub-image.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/new-packages.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/google-sheets.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/dns.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/remove-users-orm.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/documentation.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/howto/prometheus-grafana.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
   <url>
     <loc>https://docs.datahub.berkeley.edu/admins/structure.html</loc>
-    <lastmod>2024-10-24T05:10:59.359Z</lastmod>
+    <lastmod>2024-10-24T05:37:46.831Z</lastmod>
   </url>
 </urlset>
diff --git a/users/authentication.html b/users/authentication.html
deleted file mode 100644
index b19dfce44..000000000
--- a/users/authentication.html
+++ /dev/null
@@ -1,867 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
-
-<meta charset="utf-8">
-<meta name="generator" content="quarto-1.5.57">
-
-<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
-
-
-<title>User Authentication</title>
-<style>
-code{white-space: pre-wrap;}
-span.smallcaps{font-variant: small-caps;}
-div.columns{display: flex; gap: min(4vw, 1.5em);}
-div.column{flex: auto; overflow-x: auto;}
-div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
-ul.task-list{list-style: none;}
-ul.task-list li input[type="checkbox"] {
-  width: 0.8em;
-  margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ 
-  vertical-align: middle;
-}
-</style>
-
-
-<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
-<script src="../site_libs/quarto-nav/headroom.min.js"></script>
-<script src="../site_libs/clipboard/clipboard.min.js"></script>
-<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
-<script src="../site_libs/quarto-search/fuse.min.js"></script>
-<script src="../site_libs/quarto-search/quarto-search.js"></script>
-<meta name="quarto:offset" content="../">
-<link href="../admins/pre-reqs.html" rel="next">
-<link href="../users/hubs.html" rel="prev">
-<script src="../site_libs/quarto-html/quarto.js"></script>
-<script src="../site_libs/quarto-html/popper.min.js"></script>
-<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
-<script src="../site_libs/quarto-html/anchor.min.js"></script>
-<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
-<link href="../site_libs/quarto-html/quarto-syntax-highlighting.css" rel="stylesheet" id="quarto-text-highlighting-styles">
-<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
-<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
-<link href="../site_libs/bootstrap/bootstrap.min.css" rel="stylesheet" id="quarto-bootstrap" data-mode="light">
-<script id="quarto-search-options" type="application/json">{
-  "location": "navbar",
-  "copy-button": false,
-  "collapse-after": 3,
-  "panel-placement": "end",
-  "type": "overlay",
-  "limit": 50,
-  "keyboard-shortcut": [
-    "f",
-    "/",
-    "s"
-  ],
-  "show-item-context": false,
-  "language": {
-    "search-no-results-text": "No results",
-    "search-matching-documents-text": "matching documents",
-    "search-copy-link-title": "Copy link to search",
-    "search-hide-matches-text": "Hide additional matches",
-    "search-more-match-text": "more match in this document",
-    "search-more-matches-text": "more matches in this document",
-    "search-clear-button-title": "Clear",
-    "search-text-placeholder": "",
-    "search-detached-cancel-button-title": "Cancel",
-    "search-submit-button-title": "Submit",
-    "search-label": "Search"
-  }
-}</script>
-
-
-</head>
-
-<body class="nav-sidebar docked nav-fixed">
-
-<div id="quarto-search-results"></div>
-  <header id="quarto-header" class="headroom fixed-top">
-    <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
-      <div class="navbar-container container-fluid">
-      <div class="navbar-brand-container mx-auto">
-    <a href="../index.html" class="navbar-brand navbar-brand-logo">
-    <img src="../datahub.svg" alt="" class="navbar-logo">
-    </a>
-    <a class="navbar-brand" href="../index.html">
-    <span class="navbar-title">DataHub</span>
-    </a>
-  </div>
-            <div id="quarto-search" class="" title="Search"></div>
-          <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
-  <span class="navbar-toggler-icon"></span>
-</button>
-          <div class="collapse navbar-collapse" id="navbarCollapse">
-            <ul class="navbar-nav navbar-nav-scroll me-auto">
-  <li class="nav-item">
-    <a class="nav-link active" href="../admins/pre-reqs.html" aria-current="page"> 
-<span class="menu-text">Contributing</span></a>
-  </li>  
-  <li class="nav-item">
-    <a class="nav-link" href="../admins/howto/documentation.html"> 
-<span class="menu-text">Admin Tasks</span></a>
-  </li>  
-  <li class="nav-item">
-    <a class="nav-link" href="../policy/create_policy.html"> 
-<span class="menu-text">Policy</span></a>
-  </li>  
-</ul>
-          </div> <!-- /navcollapse -->
-            <div class="quarto-navbar-tools">
-    <a href="https://github.com/berkeley-dsep-infra/datahub" title="" class="quarto-navigation-tool px-1" aria-label=""><i class="bi bi-github"></i></a>
-</div>
-      </div> <!-- /container-fluid -->
-    </nav>
-  <nav class="quarto-secondary-nav">
-    <div class="container-fluid d-flex">
-      <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
-        <i class="bi bi-layout-text-sidebar-reverse"></i>
-      </button>
-        <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../users/private-repo.html">Using DataHub</a></li><li class="breadcrumb-item"><a href="../users/authentication.html">User Authentication</a></li></ol></nav>
-        <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">      
-        </a>
-    </div>
-  </nav>
-</header>
-<!-- content -->
-<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
-<!-- sidebar -->
-  <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto">
-    <div class="sidebar-menu-container"> 
-    <ul class="list-unstyled mt-1">
-        <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../index.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Home</span></a>
-  </div>
-</li>
-        <li class="sidebar-item sidebar-item-section">
-      <div class="sidebar-item-container"> 
-            <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
- <span class="menu-text">Using DataHub</span></a>
-          <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
-            <i class="bi bi-chevron-right ms-2"></i>
-          </a> 
-      </div>
-      <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">  
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/private-repo.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Accessing private GitHub repos</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">JupyterHub Deployments</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link active">
- <span class="menu-text">User Authentication</span></a>
-  </div>
-</li>
-      </ul>
-  </li>
-        <li class="sidebar-item sidebar-item-section">
-      <div class="sidebar-item-container"> 
-            <a class="sidebar-item-text sidebar-link text-start collapsed" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="false">
- <span class="menu-text">Contributing to DataHub</span></a>
-          <a class="sidebar-item-toggle text-start collapsed" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="false" aria-label="Toggle section">
-            <i class="bi bi-chevron-right ms-2"></i>
-          </a> 
-      </div>
-      <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 ">  
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/pre-reqs.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Pre-requisites</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/structure.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Repository Structure</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/storage.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User home directory storage</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/cluster-config.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Kubernetes Cluster Configuration</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/credentials.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Cloud Credentials</span></a>
-  </div>
-</li>
-          <li class="sidebar-item sidebar-item-section">
-      <div class="sidebar-item-container"> 
-            <a class="sidebar-item-text sidebar-link text-start collapsed" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="false">
- <span class="menu-text">Common Administrator Tasks</span></a>
-          <a class="sidebar-item-toggle text-start collapsed" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="false" aria-label="Toggle section">
-            <i class="bi bi-chevron-right ms-2"></i>
-          </a> 
-      </div>
-      <ul id="quarto-sidebar-section-3" class="collapse list-unstyled sidebar-section depth2 ">  
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/documentation.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Documentation</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/dns.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Update DNS</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/core-pool.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Core Node Pool Management</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/new-hub.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Create a New Hub</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/rebuild-hub-image.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Customize the Hub Docker Image</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/rebuild-postgres-image.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Customize the Per-User Postgres Docker Image</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/managing-multiple-user-image-repos.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Managing multiple user image repos</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/new-image.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Create a New Single User Image</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/transition-image.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Transition Single User Image to GitHub Actions</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/new-packages.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Testing and Upgrading New Packages</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/course-config.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Course Configuration</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/calendar-scaler.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Calendar Node Pool Autoscaler</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/prometheus-grafana.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Prometheus and Grafana</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/remove-users-orm.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">JupyterHub ORM Maintenance</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/delete-hub.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Delete or spin down a Hub</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/clusterswitch.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Switching over a hub to a new cluster</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/github-token.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Create Finely Grained Access Token</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/google-sheets.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Reading Google Sheets from DataHub</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../admins/howto/cheatsheet.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Operations Cheatsheet</span></a>
-  </div>
-</li>
-      </ul>
-  </li>
-      </ul>
-  </li>
-        <li class="sidebar-item sidebar-item-section">
-      <div class="sidebar-item-container"> 
-            <a class="sidebar-item-text sidebar-link text-start collapsed" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-4" role="navigation" aria-expanded="false">
- <span class="menu-text">Policy</span></a>
-          <a class="sidebar-item-toggle text-start collapsed" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-4" role="navigation" aria-expanded="false" aria-label="Toggle section">
-            <i class="bi bi-chevron-right ms-2"></i>
-          </a> 
-      </div>
-      <ul id="quarto-sidebar-section-4" class="collapse list-unstyled sidebar-section depth1 ">  
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../policy/create_policy.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Process to publish policy proposals</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../policy/policy_create_hubs.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Policy considerations for creating a new hub</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../policy/policy_deploy_mainhubs.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Policy considerations for deploying to the main Datahub</span></a>
-  </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../policy/principles.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Principles to be considered while creating new policies</span></a>
-  </div>
-</li>
-      </ul>
-  </li>
-        <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../incidents/index.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">Incident Reports</span></a>
-  </div>
-</li>
-    </ul>
-    </div>
-</nav>
-<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
-<!-- margin-sidebar -->
-    <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
-        <nav id="TOC" role="doc-toc" class="toc-active">
-    <h2 id="toc-title">On this page</h2>
-   
-  <ul>
-  <li><a href="#authorization" id="toc-authorization" class="nav-link active" data-scroll-target="#authorization">Authorization</a></li>
-  <li><a href="#non-berkeley-affiliates" id="toc-non-berkeley-affiliates" class="nav-link" data-scroll-target="#non-berkeley-affiliates">Non-Berkeley affiliates</a></li>
-  <li><a href="#troubleshooting" id="toc-troubleshooting" class="nav-link" data-scroll-target="#troubleshooting">Troubleshooting</a></li>
-  </ul>
-</nav>
-    </div>
-<!-- main -->
-<main class="content" id="quarto-document-content">
-
-<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../users/private-repo.html">Using DataHub</a></li><li class="breadcrumb-item"><a href="../users/authentication.html">User Authentication</a></li></ol></nav>
-<div class="quarto-title">
-<h1 class="title">User Authentication</h1>
-</div>
-
-
-
-<div class="quarto-title-meta">
-
-    
-  
-    
-  </div>
-  
-
-
-</header>
-
-
-<p>Most of our hubs use a <a href="https://www.instructure.com/canvas/">Canvas</a> instance, <a href="https://bcourses.berkeley.edu">bcourses.berkeley.edu</a>, for authentication. The hubs not using <code>CanvasOAuthenticator</code> are edx (<code>LTI1Authenticator</code>), highschool (<code>GoogleOAuthenticator</code>), and workshop (<code>dummy</code>).</p>
-<section id="authorization" class="level2">
-<h2 class="anchored" data-anchor-id="authorization">Authorization</h2>
-<p>Anyone who can log in to bCourses can log into our Canvas-based hubs. This includes all Berkeley affiliates. If you have a working berkeley.edu email account, you can most likely log in to bCourses, and hence to our JupyterHubs.</p>
-<p>Students have access for 9 months after they graduate. If they have an incomplete, they have 13 months of access instead.</p>
-</section>
-<section id="non-berkeley-affiliates" class="level2">
-<h2 class="anchored" data-anchor-id="non-berkeley-affiliates">Non-Berkeley affiliates</h2>
-<p>If someone who doesn’t have a berkeley.edu account wants to use the JupyterHubs, they need to obtain a <a href="https://calnetweb.berkeley.edu/calnet-departments/calnet-sponsored-guests">CalNet Sponsored Guest account</a> and get added to a bCourses course.</p>
-</section>
-<section id="troubleshooting" class="level2">
-<h2 class="anchored" data-anchor-id="troubleshooting">Troubleshooting</h2>
-<p>If you can log in to <a href="https://bcourses.berkeley.edu">bCourses</a> but not to any of the JupyterHubs, please contact us.</p>
-<p>If you can not log in to bCourses, please <a href="https://dls.berkeley.edu/services/bcourses-0">contact bCourses support</a>.</p>
-
-
-</section>
-
-</main> <!-- /main -->
-<script id="quarto-html-after-body" type="application/javascript">
-window.document.addEventListener("DOMContentLoaded", function (event) {
-  const toggleBodyColorMode = (bsSheetEl) => {
-    const mode = bsSheetEl.getAttribute("data-mode");
-    const bodyEl = window.document.querySelector("body");
-    if (mode === "dark") {
-      bodyEl.classList.add("quarto-dark");
-      bodyEl.classList.remove("quarto-light");
-    } else {
-      bodyEl.classList.add("quarto-light");
-      bodyEl.classList.remove("quarto-dark");
-    }
-  }
-  const toggleBodyColorPrimary = () => {
-    const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
-    if (bsSheetEl) {
-      toggleBodyColorMode(bsSheetEl);
-    }
-  }
-  toggleBodyColorPrimary();  
-  const icon = "";
-  const anchorJS = new window.AnchorJS();
-  anchorJS.options = {
-    placement: 'right',
-    icon: icon
-  };
-  anchorJS.add('.anchored');
-  const isCodeAnnotation = (el) => {
-    for (const clz of el.classList) {
-      if (clz.startsWith('code-annotation-')) {                     
-        return true;
-      }
-    }
-    return false;
-  }
-  const onCopySuccess = function(e) {
-    // button target
-    const button = e.trigger;
-    // don't keep focus
-    button.blur();
-    // flash "checked"
-    button.classList.add('code-copy-button-checked');
-    var currentTitle = button.getAttribute("title");
-    button.setAttribute("title", "Copied!");
-    let tooltip;
-    if (window.bootstrap) {
-      button.setAttribute("data-bs-toggle", "tooltip");
-      button.setAttribute("data-bs-placement", "left");
-      button.setAttribute("data-bs-title", "Copied!");
-      tooltip = new bootstrap.Tooltip(button, 
-        { trigger: "manual", 
-          customClass: "code-copy-button-tooltip",
-          offset: [0, -8]});
-      tooltip.show();    
-    }
-    setTimeout(function() {
-      if (tooltip) {
-        tooltip.hide();
-        button.removeAttribute("data-bs-title");
-        button.removeAttribute("data-bs-toggle");
-        button.removeAttribute("data-bs-placement");
-      }
-      button.setAttribute("title", currentTitle);
-      button.classList.remove('code-copy-button-checked');
-    }, 1000);
-    // clear code selection
-    e.clearSelection();
-  }
-  const getTextToCopy = function(trigger) {
-      const codeEl = trigger.previousElementSibling.cloneNode(true);
-      for (const childEl of codeEl.children) {
-        if (isCodeAnnotation(childEl)) {
-          childEl.remove();
-        }
-      }
-      return codeEl.innerText;
-  }
-  const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
-    text: getTextToCopy
-  });
-  clipboard.on('success', onCopySuccess);
-  if (window.document.getElementById('quarto-embedded-source-code-modal')) {
-    // For code content inside modals, clipBoardJS needs to be initialized with a container option
-    // TODO: Check when it could be a function (https://github.com/zenorocha/clipboard.js/issues/860)
-    const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
-      text: getTextToCopy,
-      container: window.document.getElementById('quarto-embedded-source-code-modal')
-    });
-    clipboardModal.on('success', onCopySuccess);
-  }
-    var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
-    var mailtoRegex = new RegExp(/^mailto:/);
-      var filterRegex = new RegExp("https:\/\/docs\.datahub\.berkeley\.edu");
-    var isInternal = (href) => {
-        return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
-    }
-    // Inspect non-navigation links and adorn them if external
- 	var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
-    for (var i=0; i<links.length; i++) {
-      const link = links[i];
-      if (!isInternal(link.href)) {
-        // undo the damage that might have been done by quarto-nav.js in the case of
-        // links that we want to consider external
-        if (link.dataset.originalHref !== undefined) {
-          link.href = link.dataset.originalHref;
-        }
-      }
-    }
-  function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
-    const config = {
-      allowHTML: true,
-      maxWidth: 500,
-      delay: 100,
-      arrow: false,
-      appendTo: function(el) {
-          return el.parentElement;
-      },
-      interactive: true,
-      interactiveBorder: 10,
-      theme: 'quarto',
-      placement: 'bottom-start',
-    };
-    if (contentFn) {
-      config.content = contentFn;
-    }
-    if (onTriggerFn) {
-      config.onTrigger = onTriggerFn;
-    }
-    if (onUntriggerFn) {
-      config.onUntrigger = onUntriggerFn;
-    }
-    window.tippy(el, config); 
-  }
-  const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
-  for (var i=0; i<noterefs.length; i++) {
-    const ref = noterefs[i];
-    tippyHover(ref, function() {
-      // use id or data attribute instead here
-      let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
-      try { href = new URL(href).hash; } catch {}
-      const id = href.replace(/^#\/?/, "");
-      const note = window.document.getElementById(id);
-      if (note) {
-        return note.innerHTML;
-      } else {
-        return "";
-      }
-    });
-  }
-  const xrefs = window.document.querySelectorAll('a.quarto-xref');
-  const processXRef = (id, note) => {
-    // Strip column container classes
-    const stripColumnClz = (el) => {
-      el.classList.remove("page-full", "page-columns");
-      if (el.children) {
-        for (const child of el.children) {
-          stripColumnClz(child);
-        }
-      }
-    }
-    stripColumnClz(note)
-    if (id === null || id.startsWith('sec-')) {
-      // Special case sections, only their first couple elements
-      const container = document.createElement("div");
-      if (note.children && note.children.length > 2) {
-        container.appendChild(note.children[0].cloneNode(true));
-        for (let i = 1; i < note.children.length; i++) {
-          const child = note.children[i];
-          if (child.tagName === "P" && child.innerText === "") {
-            continue;
-          } else {
-            container.appendChild(child.cloneNode(true));
-            break;
-          }
-        }
-        if (window.Quarto?.typesetMath) {
-          window.Quarto.typesetMath(container);
-        }
-        return container.innerHTML
-      } else {
-        if (window.Quarto?.typesetMath) {
-          window.Quarto.typesetMath(note);
-        }
-        return note.innerHTML;
-      }
-    } else {
-      // Remove any anchor links if they are present
-      const anchorLink = note.querySelector('a.anchorjs-link');
-      if (anchorLink) {
-        anchorLink.remove();
-      }
-      if (window.Quarto?.typesetMath) {
-        window.Quarto.typesetMath(note);
-      }
-      // TODO in 1.5, we should make sure this works without a callout special case
-      if (note.classList.contains("callout")) {
-        return note.outerHTML;
-      } else {
-        return note.innerHTML;
-      }
-    }
-  }
-  for (var i=0; i<xrefs.length; i++) {
-    const xref = xrefs[i];
-    tippyHover(xref, undefined, function(instance) {
-      instance.disable();
-      let url = xref.getAttribute('href');
-      let hash = undefined; 
-      if (url.startsWith('#')) {
-        hash = url;
-      } else {
-        try { hash = new URL(url).hash; } catch {}
-      }
-      if (hash) {
-        const id = hash.replace(/^#\/?/, "");
-        const note = window.document.getElementById(id);
-        if (note !== null) {
-          try {
-            const html = processXRef(id, note.cloneNode(true));
-            instance.setContent(html);
-          } finally {
-            instance.enable();
-            instance.show();
-          }
-        } else {
-          // See if we can fetch this
-          fetch(url.split('#')[0])
-          .then(res => res.text())
-          .then(html => {
-            const parser = new DOMParser();
-            const htmlDoc = parser.parseFromString(html, "text/html");
-            const note = htmlDoc.getElementById(id);
-            if (note !== null) {
-              const html = processXRef(id, note);
-              instance.setContent(html);
-            } 
-          }).finally(() => {
-            instance.enable();
-            instance.show();
-          });
-        }
-      } else {
-        // See if we can fetch a full url (with no hash to target)
-        // This is a special case and we should probably do some content thinning / targeting
-        fetch(url)
-        .then(res => res.text())
-        .then(html => {
-          const parser = new DOMParser();
-          const htmlDoc = parser.parseFromString(html, "text/html");
-          const note = htmlDoc.querySelector('main.content');
-          if (note !== null) {
-            // This should only happen for chapter cross references
-            // (since there is no id in the URL)
-            // remove the first header
-            if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
-              note.children[0].remove();
-            }
-            const html = processXRef(null, note);
-            instance.setContent(html);
-          } 
-        }).finally(() => {
-          instance.enable();
-          instance.show();
-        });
-      }
-    }, function(instance) {
-    });
-  }
-      let selectedAnnoteEl;
-      const selectorForAnnotation = ( cell, annotation) => {
-        let cellAttr = 'data-code-cell="' + cell + '"';
-        let lineAttr = 'data-code-annotation="' +  annotation + '"';
-        const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
-        return selector;
-      }
-      const selectCodeLines = (annoteEl) => {
-        const doc = window.document;
-        const targetCell = annoteEl.getAttribute("data-target-cell");
-        const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
-        const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
-        const lines = annoteSpan.getAttribute("data-code-lines").split(",");
-        const lineIds = lines.map((line) => {
-          return targetCell + "-" + line;
-        })
-        let top = null;
-        let height = null;
-        let parent = null;
-        if (lineIds.length > 0) {
-            //compute the position of the single el (top and bottom and make a div)
-            const el = window.document.getElementById(lineIds[0]);
-            top = el.offsetTop;
-            height = el.offsetHeight;
-            parent = el.parentElement.parentElement;
-          if (lineIds.length > 1) {
-            const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
-            const bottom = lastEl.offsetTop + lastEl.offsetHeight;
-            height = bottom - top;
-          }
-          if (top !== null && height !== null && parent !== null) {
-            // cook up a div (if necessary) and position it 
-            let div = window.document.getElementById("code-annotation-line-highlight");
-            if (div === null) {
-              div = window.document.createElement("div");
-              div.setAttribute("id", "code-annotation-line-highlight");
-              div.style.position = 'absolute';
-              parent.appendChild(div);
-            }
-            div.style.top = top - 2 + "px";
-            div.style.height = height + 4 + "px";
-            div.style.left = 0;
-            let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
-            if (gutterDiv === null) {
-              gutterDiv = window.document.createElement("div");
-              gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
-              gutterDiv.style.position = 'absolute';
-              const codeCell = window.document.getElementById(targetCell);
-              const gutter = codeCell.querySelector('.code-annotation-gutter');
-              gutter.appendChild(gutterDiv);
-            }
-            gutterDiv.style.top = top - 2 + "px";
-            gutterDiv.style.height = height + 4 + "px";
-          }
-          selectedAnnoteEl = annoteEl;
-        }
-      };
-      const unselectCodeLines = () => {
-        const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
-        elementsIds.forEach((elId) => {
-          const div = window.document.getElementById(elId);
-          if (div) {
-            div.remove();
-          }
-        });
-        selectedAnnoteEl = undefined;
-      };
-        // Handle positioning of the toggle
-    window.addEventListener(
-      "resize",
-      throttle(() => {
-        elRect = undefined;
-        if (selectedAnnoteEl) {
-          selectCodeLines(selectedAnnoteEl);
-        }
-      }, 10)
-    );
-    function throttle(fn, ms) {
-    let throttle = false;
-    let timer;
-      return (...args) => {
-        if(!throttle) { // first call gets through
-            fn.apply(this, args);
-            throttle = true;
-        } else { // all the others get throttled
-            if(timer) clearTimeout(timer); // cancel #2
-            timer = setTimeout(() => {
-              fn.apply(this, args);
-              timer = throttle = false;
-            }, ms);
-        }
-      };
-    }
-      // Attach click handler to the DT
-      const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
-      for (const annoteDlNode of annoteDls) {
-        annoteDlNode.addEventListener('click', (event) => {
-          const clickedEl = event.target;
-          if (clickedEl !== selectedAnnoteEl) {
-            unselectCodeLines();
-            const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
-            if (activeEl) {
-              activeEl.classList.remove('code-annotation-active');
-            }
-            selectCodeLines(clickedEl);
-            clickedEl.classList.add('code-annotation-active');
-          } else {
-            // Unselect the line
-            unselectCodeLines();
-            clickedEl.classList.remove('code-annotation-active');
-          }
-        });
-      }
-  const findCites = (el) => {
-    const parentEl = el.parentElement;
-    if (parentEl) {
-      const cites = parentEl.dataset.cites;
-      if (cites) {
-        return {
-          el,
-          cites: cites.split(' ')
-        };
-      } else {
-        return findCites(el.parentElement)
-      }
-    } else {
-      return undefined;
-    }
-  };
-  var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
-  for (var i=0; i<bibliorefs.length; i++) {
-    const ref = bibliorefs[i];
-    const citeInfo = findCites(ref);
-    if (citeInfo) {
-      tippyHover(citeInfo.el, function() {
-        var popup = window.document.createElement('div');
-        citeInfo.cites.forEach(function(cite) {
-          var citeDiv = window.document.createElement('div');
-          citeDiv.classList.add('hanging-indent');
-          citeDiv.classList.add('csl-entry');
-          var biblioDiv = window.document.getElementById('ref-' + cite);
-          if (biblioDiv) {
-            citeDiv.innerHTML = biblioDiv.innerHTML;
-          }
-          popup.appendChild(citeDiv);
-        });
-        return popup.innerHTML;
-      });
-    }
-  }
-});
-</script>
-<nav class="page-navigation">
-  <div class="nav-page nav-page-previous">
-      <a href="../users/hubs.html" class="pagination-link" aria-label="JupyterHub Deployments">
-        <i class="bi bi-arrow-left-short"></i> <span class="nav-page-text">JupyterHub Deployments</span>
-      </a>          
-  </div>
-  <div class="nav-page nav-page-next">
-      <a href="../admins/pre-reqs.html" class="pagination-link" aria-label="Pre-requisites">
-        <span class="nav-page-text">Pre-requisites</span> <i class="bi bi-arrow-right-short"></i>
-      </a>
-  </div>
-</nav>
-</div> <!-- /content -->
-
-
-
-
-</body></html>
\ No newline at end of file
diff --git a/users/hubs.html b/users/hubs.html
index 16aa5732d..172680a0f 100644
--- a/users/hubs.html
+++ b/users/hubs.html
@@ -30,7 +30,7 @@
 <script src="../site_libs/quarto-search/fuse.min.js"></script>
 <script src="../site_libs/quarto-search/quarto-search.js"></script>
 <meta name="quarto:offset" content="../">
-<link href="../users/authentication.html" rel="next">
+<link href="../admins/pre-reqs.html" rel="next">
 <link href="../users/private-repo.html" rel="prev">
 <script src="../site_libs/quarto-listing/list.min.js"></script>
 <script src="../site_libs/quarto-listing/quarto-listing.js"></script>
@@ -214,12 +214,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link active">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
@@ -466,7 +460,7 @@ <h1 class="title">JupyterHub Deployments</h1>
 
 <div class="quarto-listing quarto-listing-container-default" id="listing-listing">
 <div class="list quarto-listing-default">
-<div class="quarto-post image-right" data-index="0" data-categories="course,jupyterlab,otter-grader,shared-folders" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="72">
+<div class="quarto-post image-right" data-index="0" data-categories="course,jupyterlab,otter-grader,shared-folders" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="72">
 <div class="thumbnail">
 <p><a href="../users/hubs/data100.html" class="no-external"></a></p><a href="../users/hubs/data100.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -502,7 +496,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="1" data-categories="course,jupyterlab,otter-grader,shared-folders" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="30">
+<div class="quarto-post image-right" data-index="1" data-categories="course,jupyterlab,otter-grader,shared-folders" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="30">
 <div class="thumbnail">
 <p><a href="../users/hubs/data102.html" class="no-external"></a></p><a href="../users/hubs/data102.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -538,7 +532,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="2" data-categories="jupyterlab,ltiauthenticator" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="42">
+<div class="quarto-post image-right" data-index="2" data-categories="jupyterlab,ltiauthenticator" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="42">
 <div class="thumbnail">
 <p><a href="../users/hubs/edx.html" class="no-external"></a></p><a href="../users/hubs/edx.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -568,7 +562,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="3" data-categories="jupyterlab,r,vscode" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="53">
+<div class="quarto-post image-right" data-index="3" data-categories="jupyterlab,r,vscode" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="53">
 <div class="thumbnail">
 <p><a href="../users/hubs/datahub.html" class="no-external"></a></p><a href="../users/hubs/datahub.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -601,7 +595,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="4" data-categories="course,jupyterlab,vscode" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="44">
+<div class="quarto-post image-right" data-index="4" data-categories="course,jupyterlab,vscode" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="44">
 <div class="thumbnail">
 <p><a href="../users/hubs/prob140.html" class="no-external"></a></p><a href="../users/hubs/prob140.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -634,7 +628,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="5" data-categories="jupyterlab,r,rstudio,vscode" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="34">
+<div class="quarto-post image-right" data-index="5" data-categories="jupyterlab,r,rstudio,vscode" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="34">
 <div class="thumbnail">
 <p><a href="../users/hubs/r.html" class="no-external"></a></p><a href="../users/hubs/r.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -670,7 +664,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="6" data-categories="r,rstudio,shiny" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="16">
+<div class="quarto-post image-right" data-index="6" data-categories="r,rstudio,shiny" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="16">
 <div class="thumbnail">
 <p><a href="../users/hubs/shiny.html" class="no-external"></a></p><a href="../users/hubs/shiny.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -703,7 +697,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="7" data-categories="course,jupyterlab,rtc,shared-folders,vscode" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="2" data-listing-word-count-sort="318">
+<div class="quarto-post image-right" data-index="7" data-categories="course,jupyterlab,rtc,shared-folders,vscode" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="2" data-listing-word-count-sort="318">
 <div class="thumbnail">
 <p><a href="../users/hubs/stat159.html" class="no-external"></a></p><a href="../users/hubs/stat159.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -742,7 +736,7 @@ <h3 class="no-anchor listing-title">
 </a>
 </div>
 </div>
-<div class="quarto-post image-right" data-index="8" data-categories="course,jupyterlab,quarto,rstudio,shared-folders,shiny" data-listing-file-modified-sort="1729746659359" data-listing-reading-time-sort="1" data-listing-word-count-sort="33">
+<div class="quarto-post image-right" data-index="8" data-categories="course,jupyterlab,quarto,rstudio,shared-folders,shiny" data-listing-file-modified-sort="1729748266831" data-listing-reading-time-sort="1" data-listing-word-count-sort="33">
 <div class="thumbnail">
 <p><a href="../users/hubs/stat20.html" class="no-external"></a></p><a href="../users/hubs/stat20.html" class="no-external">
 <div class="listing-item-img-placeholder card-img-top" >&nbsp;</div>
@@ -1216,8 +1210,8 @@ <h3 class="no-anchor listing-title">
       </a>          
   </div>
   <div class="nav-page nav-page-next">
-      <a href="../users/authentication.html" class="pagination-link" aria-label="User Authentication">
-        <span class="nav-page-text">User Authentication</span> <i class="bi bi-arrow-right-short"></i>
+      <a href="../admins/pre-reqs.html" class="pagination-link" aria-label="Pre-requisites">
+        <span class="nav-page-text">Pre-requisites</span> <i class="bi bi-arrow-right-short"></i>
       </a>
   </div>
 </nav>
diff --git a/users/hubs/data100.html b/users/hubs/data100.html
index 9885d770f..ef8438962 100644
--- a/users/hubs/data100.html
+++ b/users/hubs/data100.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/data102.html b/users/hubs/data102.html
index d5d059e35..9826112a1 100644
--- a/users/hubs/data102.html
+++ b/users/hubs/data102.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/datahub.html b/users/hubs/datahub.html
index e62f4d4b7..b8054904e 100644
--- a/users/hubs/datahub.html
+++ b/users/hubs/datahub.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/edx.html b/users/hubs/edx.html
index 44fdca3c8..6a5260643 100644
--- a/users/hubs/edx.html
+++ b/users/hubs/edx.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/prob140.html b/users/hubs/prob140.html
index a141048bf..904ee381d 100644
--- a/users/hubs/prob140.html
+++ b/users/hubs/prob140.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/r.html b/users/hubs/r.html
index 0a5974ffc..7b0ca62bf 100644
--- a/users/hubs/r.html
+++ b/users/hubs/r.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/shiny.html b/users/hubs/shiny.html
index 7adae02e7..28cb3dc96 100644
--- a/users/hubs/shiny.html
+++ b/users/hubs/shiny.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/stat159.html b/users/hubs/stat159.html
index bde9adcbf..c8d5334be 100644
--- a/users/hubs/stat159.html
+++ b/users/hubs/stat159.html
@@ -186,12 +186,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/hubs/stat20.html b/users/hubs/stat20.html
index 49ae68973..c32134d3a 100644
--- a/users/hubs/stat20.html
+++ b/users/hubs/stat20.html
@@ -152,12 +152,6 @@
   <a href="../../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/index.html b/users/index.html
index b9c7178fd..b38de8684 100644
--- a/users/index.html
+++ b/users/index.html
@@ -152,12 +152,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>
diff --git a/users/private-repo.html b/users/private-repo.html
index a7984ef9f..65067de0f 100644
--- a/users/private-repo.html
+++ b/users/private-repo.html
@@ -154,12 +154,6 @@
   <a href="../users/hubs.html" class="sidebar-item-text sidebar-link">
  <span class="menu-text">JupyterHub Deployments</span></a>
   </div>
-</li>
-          <li class="sidebar-item">
-  <div class="sidebar-item-container"> 
-  <a href="../users/authentication.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">User Authentication</span></a>
-  </div>
 </li>
       </ul>
   </li>