Add Kubernetes Upgrade ISSUE_TEMPLATE (#5310)

* Add Kubernetes Upgrade ISSUE_TEMPLATE

* Merge prod cluster upgrade

* Add script to generate issues from upgrade epic ticket

* Update about info and add label

* Add planning and updating runbook steps

.github/ISSUE_TEMPLATE/cloud-platform-k8s-upgrade-template.md

@@ -0,0 +1,193 @@
+---
+name: Cloud Platform Kubernetes Upgrade story
+about: This template is for Cloud Platform team to create an epic ticket for kubernetes upgrades
+label: 'EPIC'
+---
+
+## Issue 1:
+### Planning upgrade to EKS <version>
+
+Go through the release notes of EKS 1.27 and create a plan to upgrade our clusters
+
+Things to consider:
+
+Review changelog & release notes
+EKS Module support at current version?
+Are there any API deprecations & removals? (Check insights)
+Are there new components being added?
+What changes are being introduced to current components?
+Are there changes to core infra of the CP required? i.e. Are all our current components compatible with <version>?
+Are there changes users need to make?
+Do we need to expand any of our smoke/integration testing?
+Create additional tickets needed for any findings specific to this upgrade
+
+Cluster upgrade Runbook:
+https://runbooks.cloud-platform.service.justice.gov.uk/upgrade-eks-cluster.html
+
+## Issue 2:
+### Update vpc-cni from <current-version> to the version needed for current k8s version
+Check for the latest supported addon version for the current kubernetes version and update the clusters
+
+Production Clusters Checklist:
+
+- [ ] live-2
+- [ ] manager
+- [ ] live
+
+See the [Amazon EKS add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) documentation for more information about addons, or find the latest versions for these EKS add-ons directly:
+
+- [coredns](https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html)
+- [kube-proxy](https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html)
+- [vpc-cni](https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html)
+
+
+## Issue 3:
+### Update kube-proxy from <current-version> to the version needed for current k8s version
+Check for the latest supported addon version for the current kubernetes version and update the clusters
+
+Production Clusters Checklist:
+
+- [ ] live-2
+- [ ] manager
+- [ ] live
+
+See the [Amazon EKS add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) documentation for more information about addons, or find the latest versions for these EKS add-ons directly:
+
+- [coredns](https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html)
+- [kube-proxy](https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html)
+- [vpc-cni](https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html)
+
+
+## Issue 4:
+### Update core-dns from <current-version> to the version needed for current k8s version
+Check for the latest supported addon version for the current kubernetes version and update the clusters
+
+Production Clusters Checklist:
+
+- [ ] live-2
+- [ ] manager
+- [ ] live
+
+See the [Amazon EKS add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) documentation for more information about addons, or find the latest versions for these EKS add-ons directly:
+
+- [coredns](https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html)
+- [kube-proxy](https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html)
+- [vpc-cni](https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html)
+
+## Issue 5:
+### Test EKS <version> on test cluster
+
+Follow instructions from Upgrade runbook: https://runbooks.cloud-platform.service.justice.gov.uk/upgrade-eks-cluster.html#upgrade-eks-cluster
+
+## Issue 6:
+### Test EKS <version> on live-like cluster
+
+Follow instructions from creating a live like cluster runbook: https://runbooks.cloud-platform.service.justice.gov.uk/creating-a-live-like.html#creating-a-live-like-cluster
+
+## Issue 7:
+### EKS: Upgrade Production clusters to Kubernetes <version>
+https://runbooks.cloud-platform.service.justice.gov.uk/upgrade-eks-cluster.html
+
+Production Clusters Checklist:
+
+- [ ] live-2
+- [ ] manager
+- [ ] live
+
+## Issue 8:
+### Update vpc-cni from <version> to the version needed for the upgraded k8s version
+Check for the latest supported addon version for the upgraded kubernetes version and update the clusters
+
+Production Clusters Checklist:
+
+- [ ] live-2
+- [ ] manager
+- [ ] live
+
+See the [Amazon EKS add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) documentation for more information about addons, or find the latest versions for these EKS add-ons directly:
+
+- [coredns](https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html)
+- [kube-proxy](https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html)
+- [vpc-cni](https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html)
+
+
+## Issue 9:
+### Update kube-proxy from <version> to the version needed for the upgraded k8s version
+Check for the latest supported addon version for the upgraded kubernetes version and update the clusters
+
+Production Clusters Checklist:
+
+- [ ] live-2
+- [ ] manager
+- [ ] live
+
+See the [Amazon EKS add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) documentation for more information about addons, or find the latest versions for these EKS add-ons directly:
+
+- [coredns](https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html)
+- [kube-proxy](https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html)
+- [vpc-cni](https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html)
+
+
+## Issue 10:
+### Update core-dns from <version> to the version needed for the upgraded k8s version
+Check for the latest supported addon version for the upgraded kubernetes version and update the clusters
+
+Production Clusters Checklist:
+
+- [ ] live-2
+- [ ] manager
+- [ ] live
+
+See the [Amazon EKS add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) documentation for more information about addons, or find the latest versions for these EKS add-ons directly:
+
+- [coredns](https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html)
+- [kube-proxy](https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html)
+- [vpc-cni](https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html)
+
+## Issue 11:
+### Review cluster components for upgrading
+https://runbooks.cloud-platform.service.justice.gov.uk/container-images.html#container-images-used-by-cluster-components
+
+Review the compatibility matrix for all cluster components and verify if the image is compatible with the upgraded kuebrnetes version
+
+## Issue 12:
+### Review and upgrade kube-state-metrics for the upgraded cluster version <version>
+https://runbooks.cloud-platform.service.justice.gov.uk/container-images.html#container-images-used-by-cluster-components
+
+https://github.com/kubernetes/kube-state-metrics?tab=readme-ov-file#compatibility-matrix
+
+Review the compatibility matrix of kube-state-metrics and verify if the image is compatible with the upgraded kuebrnetes version. If the version is mismatch, check if the whole kube-prometheus-chart needs updating to get the default version of kube-state-metrics from the chart-> values. Otherwise, pin the kube-state-metrics image to match the kubernetes-version
+
+## Issue 13:
+### Upgrade cluster autoscalar for k8s version <version>
+The Cloud Platform Cluster is in k8s version <version>. Hence upgrade the cluster-autoscalar to match the k8s version.
+
+https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler
+https://github.com/ministryofjustice/cloud-platform-terraform-cluster-autoscaler
+
+## Issue 14:
+### Upgrade cluster descheudler for k8s version <version>
+The Cloud Platform Cluster is in k8s version <version>. Hence upgrade the descheduler to match the k8s version.
+
+https://github.com/kubernetes-sigs/descheduler?tab=readme-ov-file#%EF%B8%8F--documentation-versions-by-release
+https://github.com/ministryofjustice/cloud-platform-terraform-descheduler
+
+## Issue 15:
+### Post k8s version <version> Cleanup
+Following upgrade from EKS <old-version > <version>, there will be a number of cleanup activities that need addressing:
+
+Update user guide / runbook references
+Update tools-image for kubectl version
+Update cloud-platform-cli for kubectl version 
+Update concourse pipelines to use updated tools-image and cli
+
+## Issue 16:
+### Update Upgrade runbook and Issue template:
+
+Update the runbook for 
+- any changes needed in the steps to perform the upgrade
+- any lessons learnt that could be useful for next upgrade
+- any changes to the upgrade issue template
+
+Cluster upgrade Runbook:
+https://runbooks.cloud-platform.service.justice.gov.uk/upgrade-eks-cluster.html

cmd/create-upgrade-issues/go.mod

@@ -0,0 +1,17 @@
+module ministryofjustice/cloud-platform/cmd/create-upgrade-issues
+
+go 1.21.7
+
+require (
+	github.com/google/go-github/v35 v35.3.0
+	golang.org/x/oauth2 v0.17.0
+)
+
+require (
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+)

cmd/create-upgrade-issues/go.sum

@@ -0,0 +1,35 @@
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho=
+github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
+golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=

cmd/create-upgrade-issues/main.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/google/go-github/v35/github"
+	"golang.org/x/oauth2"
+)
+
+// GitHubAccess contains GitHub access information
+type GitHubAccess struct {
+	AccessToken string
+	RepoOwner   string
+	RepoName    string
+	IssueNumber int
+}
+
+// ParseIssue parses the given issue and returns a slice of issues
+func ParseIssue(issue string) []string {
+	var issues []string
+	sections := strings.Split(issue, "## Issue")
+
+	for _, section := range sections {
+		if section != "" {
+			lines := strings.Split(strings.TrimSpace(section), "\n")
+			title := strings.TrimSpace(lines[0])
+			body := strings.TrimSpace(strings.Join(lines[1:], "\n"))
+			issues = append(issues, fmt.Sprintf("## %s\n%s", title, body))
+		}
+	}
+	return issues
+}
+
+// CreateIssue creates a GitHub issue
+func CreateIssue(client *github.Client, ghAccess GitHubAccess, issue string, epicIssueNumber int) error {
+	issueParts := strings.SplitN(issue, "\n", 3)
+	title := strings.TrimSpace(strings.TrimPrefix(issueParts[1], "###"))
+	body := strings.TrimSpace(issueParts[2])
+
+	// Append the body with the epic issue number
+	body = fmt.Sprintf("%s\n\n Related to: #%d", body, epicIssueNumber)
+
+	issueRequest := &github.IssueRequest{
+		Title: &title,
+		Body:  &body,
+	}
+	_, _, err := client.Issues.Create(context.Background(), ghAccess.RepoOwner, ghAccess.RepoName, issueRequest)
+	return err
+}
+
+func main() {
+
+	var ghAccess GitHubAccess
+	ghAccess.AccessToken = os.Getenv("GITHUB_TOKEN")
+	flag.StringVar(&ghAccess.RepoOwner, "owner", "ministryofjustice", "the repository to create issues")
+	flag.StringVar(&ghAccess.RepoName, "repo", "cloud-platform", "the repository to create issues")
+	flag.IntVar(&ghAccess.IssueNumber, "issue", 0, "the issue number to create issues")
+
+	flag.Parse()
+
+	ctx := context.Background()
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: ghAccess.AccessToken},
+	)
+	tc := oauth2.NewClient(ctx, ts)
+	client := github.NewClient(tc)
+
+	// Fetch the issue contents from GitHub
+	issueContent, _, err := client.Issues.Get(ctx, ghAccess.RepoOwner, ghAccess.RepoName, ghAccess.IssueNumber)
+	if err != nil {
+		fmt.Println("Error fetching issue template:", err)
+		os.Exit(1)
+	}
+
+	issues := ParseIssue(*issueContent.Body)
+	for _, issue := range issues {
+		//fmt.Println("Creating issue:", issue)
+		err := CreateIssue(client, ghAccess, issue, ghAccess.IssueNumber)
+		if err != nil {
+			fmt.Println("Error creating issue:", err)
+			os.Exit(1)
+		}
+	}
+}