Skip to content

Commit

Permalink
fix: refactor code for vault, http and k8s client (#39)
Browse files Browse the repository at this point in the history
  • Loading branch information
@samirtahir91
samirtahir91 authored Apr 13, 2024
1 parent bd7fcd1 commit 2fee4e8
Show file tree
Hide file tree
Showing 5 changed files with 132 additions and 130 deletions.
29 changes: 26 additions & 3 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@ import (

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
vault "github.com/hashicorp/vault/api" // vault client
kubernetes "k8s.io/client-go/kubernetes" // k8s client
_ "k8s.io/client-go/plugin/pkg/client/auth"
"net/http" // http client
ctrlConfig "sigs.k8s.io/controller-runtime/pkg/client/config"

"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
Expand Down Expand Up @@ -101,6 +105,22 @@ func main() {
TLSOpts: tlsOpts,
})

// http client
httpClient := &http.Client{}

// Initialise vault client with VAULT_ADDRESS env var
vaultAddress := os.Getenv("VAULT_ADDRESS") // Vault server fqdn
vaultClient, err := vault.NewClient(&vault.Config{
Address: vaultAddress,
})
if err != nil {
setupLog.Error(err, "failed to initialise Vault client")
os.Exit(1)
}

// Initialise K8s client
k8sClientset := kubernetes.NewForConfigOrDie(ctrlConfig.GetConfigOrDie())

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{
Expand Down Expand Up @@ -130,9 +150,12 @@ func main() {
}

if err = (&controller.GithubAppReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("githubapp-controller"),
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("githubapp-controller"),
HTTPClient: httpClient,
VaultClient: vaultClient,
K8sClient: k8sClientset,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "GithubApp")
os.Exit(1)
Expand Down
92 changes: 72 additions & 20 deletions internal/controller/githubapp_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,14 @@ import (
"time"

githubappv1 "github-app-operator/api/v1"
vault "github.com/hashicorp/vault/api" // vault client
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
kubernetes "k8s.io/client-go/kubernetes" // k8s client
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder" // Required for Watching
Expand All @@ -46,19 +48,23 @@ import (
// GithubAppReconciler reconciles a GithubApp object
type GithubAppReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
lock sync.Mutex
Scheme *runtime.Scheme
Recorder record.EventRecorder
HTTPClient *http.Client
VaultClient *vault.Client
K8sClient *kubernetes.Clientset
lock sync.Mutex
}

var (
defaultRequeueAfter = 5 * time.Minute // Default requeue interval
defaultTimeBeforeExpiry = 15 * time.Minute // Default time before expiry
reconcileInterval time.Duration // Requeue interval (from env var)
timeBeforeExpiry time.Duration // Expiry threshold (from env var)
vaultAddress = os.Getenv("VAULT_ADDRESS") // Vault server fqdn
vaultAudience = os.Getenv("VAULT_ROLE_AUDIENCE") // Vault audience bound to role
vaultRole = os.Getenv("VAULT_ROLE") // Vault role to use
serviceAccountName string // controller service account
kubernetesNamespace string // controller namespace
)

const (
Expand Down Expand Up @@ -224,7 +230,7 @@ func (r *GithubAppReconciler) checkExpiryAndUpdateAccessToken(ctx context.Contex
username := string(accessTokenSecret.Data["username"])

// Check if the access token is a valid github token via gh api auth
if !isAccessTokenValid(ctx, username, accessToken) {
if !r.isAccessTokenValid(ctx, username, accessToken) {
// If accessToken is invalid, generate or update access token
return r.generateOrUpdateAccessToken(ctx, githubApp)
}
Expand All @@ -245,7 +251,7 @@ func (r *GithubAppReconciler) checkExpiryAndUpdateAccessToken(ctx context.Contex
}

// Function to check if the access token is valid by making a request to GitHub API
func isAccessTokenValid(ctx context.Context, username string, accessToken string) bool {
func (r *GithubAppReconciler) isAccessTokenValid(ctx context.Context, username string, accessToken string) bool {
l := log.FromContext(ctx)

// If username has been modified, renew the secret
Expand All @@ -259,9 +265,6 @@ func isAccessTokenValid(ctx context.Context, username string, accessToken string
// GitHub API endpoint for rate limit information
url := "https://api.github.com/rate_limit"

// Create a new HTTP client
client := &http.Client{}

// Create a new request
ghReq, err := http.NewRequest("GET", url, nil)
if err != nil {
Expand All @@ -273,7 +276,7 @@ func isAccessTokenValid(ctx context.Context, username string, accessToken string
ghReq.Header.Set("Authorization", "token "+accessToken)

// Send the request
resp, err := client.Do(ghReq)
resp, err := r.HTTPClient.Do(ghReq)
if err != nil {
l.Error(err, "Error sending request to GitHub API for rate limit")
return false
Expand Down Expand Up @@ -359,16 +362,16 @@ func (r *GithubAppReconciler) getPrivateKeyFromSecret(ctx context.Context, githu
}

// Function to get private key from a Vault secret
func getPrivateKeyFromVault(ctx context.Context, mountPath string, secretPath string, secretKey string) ([]byte, error) {
func (r *GithubAppReconciler) getPrivateKeyFromVault(ctx context.Context, mountPath string, secretPath string, secretKey string) ([]byte, error) {

// Get JWT from k8s Token Request API
token, err := RequestToken(ctx, vaultAudience)
token, err := r.RequestToken(ctx, vaultAudience, kubernetesNamespace, serviceAccountName)
if err != nil {
return []byte(""), err
}

// Get private key from Vault secret with short-lived JWT
privateKey, err := GetSecretWithKubernetesAuth(token, vaultAddress, vaultRole, mountPath, secretPath, secretKey)
privateKey, err := r.GetSecretWithKubernetesAuth(token, vaultRole, mountPath, secretPath, secretKey)
if err != nil {
return []byte(""), err
}
Expand All @@ -386,14 +389,14 @@ func (r *GithubAppReconciler) generateOrUpdateAccessToken(ctx context.Context, g
// Vault auth will take precedence over using `spec.privateKeySecret`
if githubApp.Spec.VaultPrivateKey != nil {

if vaultAddress == "" || vaultAudience == "" || vaultRole == "" {
if r.VaultClient.Address() == "" || vaultAudience == "" || vaultRole == "" {
return fmt.Errorf("failed on vault auth: VAULT_ROLE, VAULT_ROLE_AUDIENCE and VAULT_ADDRESS are required env variables for Vault authentication")
}

mountPath := githubApp.Spec.VaultPrivateKey.MountPath
secretPath := githubApp.Spec.VaultPrivateKey.SecretPath
secretKey := githubApp.Spec.VaultPrivateKey.SecretKey
privateKey, privateKeyErr = getPrivateKeyFromVault(ctx, mountPath, secretPath, secretKey)
privateKey, privateKeyErr = r.getPrivateKeyFromVault(ctx, mountPath, secretPath, secretKey)
if privateKeyErr != nil {
return fmt.Errorf("failed to get private key from vault: %v", privateKeyErr)
}
Expand All @@ -406,7 +409,7 @@ func (r *GithubAppReconciler) generateOrUpdateAccessToken(ctx context.Context, g
}

// Generate or renew access token
accessToken, expiresAt, err := generateAccessToken(
accessToken, expiresAt, err := r.generateAccessToken(
ctx,
githubApp.Spec.AppId,
githubApp.Spec.InstallId,
Expand Down Expand Up @@ -556,7 +559,7 @@ func updateGithubAppStatusWithRetry(ctx context.Context, r *GithubAppReconciler,
}

// function to generate new access token for gh app
func generateAccessToken(ctx context.Context, appID int, installationID int, privateKey []byte) (string, metav1.Time, error) {
func (r *GithubAppReconciler) generateAccessToken(ctx context.Context, appID int, installationID int, privateKey []byte) (string, metav1.Time, error) {

l := log.FromContext(ctx)

Expand All @@ -579,8 +582,7 @@ func generateAccessToken(ctx context.Context, appID int, installationID int, pri
return "", metav1.Time{}, fmt.Errorf("failed to sign JWT: %v", err)
}

// Create HTTP client and perform request to get installation token
httpClient := &http.Client{}
// Use HTTP client and perform request to get installation token
url := fmt.Sprintf("https://api.github.com/app/installations/%d/access_tokens", installationID)
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, nil)
if err != nil {
Expand All @@ -590,7 +592,7 @@ func generateAccessToken(ctx context.Context, appID int, installationID int, pri
req.Header.Set("Accept", "application/vnd.github+json")

// Send post request for access token
resp, err := httpClient.Do(req)
resp, err := r.HTTPClient.Do(req)
if err != nil {
return "", metav1.Time{}, fmt.Errorf("failed to perform HTTP request: %v", err)
}
Expand Down Expand Up @@ -722,6 +724,48 @@ func githubAppPredicate() predicate.Predicate {
}
}

// Function to get service account and namespace of controller
func getServiceAccountAndNamespace() (string, string, error) {

// Get KSA mounted in pod
serviceAccountToken, err := os.ReadFile("var/run/secrets/kubernetes.io/serviceaccount/token")
if err != nil {
return "", "", fmt.Errorf("failed to read service account token: %v", err)
}
// Parse the KSA token
parsedToken, _, err := new(jwt.Parser).ParseUnverified(string(serviceAccountToken), jwt.MapClaims{})
if err != nil {
return "", "", fmt.Errorf("failed to parse token: %v", err)
}
// Get the claims
claims, ok := parsedToken.Claims.(jwt.MapClaims)
if !ok {
return "", "", fmt.Errorf("failed to parse token claims")
}
// Get kubernetes.io claims
kubernetesClaims, ok := claims["kubernetes.io"].(map[string]interface{})
if !ok {
return "", "", fmt.Errorf("failed to assert kubernetes.io claim to map[string]interface{}")
}
// Get serviceaccount claim
serviceAccountClaims, ok := kubernetesClaims["serviceaccount"].(map[string]interface{})
if !ok {
return "", "", fmt.Errorf("failed to assert serviceaccount claim to map[string]interface{}")
}
// Get the namespace
kubernetesNamespace, ok := kubernetesClaims["namespace"].(string)
if !ok {
return "", "", fmt.Errorf("failed to assert namespace to string")
}
// Get service account name
serviceAccountName, ok := serviceAccountClaims["name"].(string)
if !ok {
return "", "", fmt.Errorf("failed to assert service account name to string")
}

return serviceAccountName, kubernetesNamespace, nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *GithubAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
// Get reconcile interval from environment variable or use default value
Expand All @@ -743,6 +787,14 @@ func (r *GithubAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
timeBeforeExpiry = defaultTimeBeforeExpiry
}

// Get service account name and namespace
serviceAccountName, kubernetesNamespace, err = getServiceAccountAndNamespace()
if err != nil {
log.Log.Error(err, "failed to get service account and/or namespace of controller")
} else {
log.Log.Info("Got controller aervice account and namespace", "service account", serviceAccountName, "namespace", kubernetesNamespace)
}

return ctrl.NewControllerManagedBy(mgr).
// Watch GithubApps
For(&githubappv1.GithubApp{}, builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}, githubAppPredicate())).
Expand Down
30 changes: 0 additions & 30 deletions internal/controller/githubapp_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

const (
Expand Down Expand Up @@ -147,35 +146,6 @@ var _ = Describe("GithubApp controller", func() {
})
})

Context("When requeing a reconcile for a GithubApp that is not expired", func() {
It("should successfully reconcile the resource and get the rate limit", func() {
ctx := context.Background()

By("Reconciling the created resource")
controllerReconciler := &GithubAppReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
}

// Perform reconciliation for the resource
result, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: namespace1,
Name: githubAppName,
},
})

// Verify if reconciliation was successful
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Reconciliation failed: %v", err))

// Print the result
fmt.Println("Reconciliation result:", result)

// Delete the GitHubApp after reconciliation
test_helpers.DeleteGitHubAppAndWait(ctx, k8sClient, namespace1, githubAppName)
})
})

Context("When reconciling a GithubApp with spec.rolloutDeployment.labels.foo as bar", func() {
It("Should eventually upgrade the deployment matching label foo: bar", func() {
if os.Getenv("USE_EXISTING_CLUSTER") == "" {
Expand Down
29 changes: 21 additions & 8 deletions internal/controller/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@ package controller
import (
"context"
"fmt"
"net/http" // http client
"os"
"path/filepath"
"runtime"
"testing"

vault "github.com/hashicorp/vault/api" // vault client
kubernetes "k8s.io/client-go/kubernetes" // k8s client

ctrl "sigs.k8s.io/controller-runtime"

. "github.com/onsi/ginkgo/v2"
Expand All @@ -44,11 +48,14 @@ import (
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.

var (
cfg *rest.Config
k8sClient client.Client
testEnv *envtest.Environment
ctx context.Context
cancel context.CancelFunc
cfg *rest.Config
k8sClient client.Client
httpClient *http.Client
vaultClient *vault.Client
k8sClientset *kubernetes.Clientset
testEnv *envtest.Environment
ctx context.Context
cancel context.CancelFunc
)

func TestControllers(t *testing.T) {
Expand Down Expand Up @@ -104,10 +111,16 @@ var _ = BeforeSuite(func() {
})
Expect(err).ToNot(HaveOccurred())

// http client
httpClient = &http.Client{}

err = (&GithubAppReconciler{
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),
Recorder: k8sManager.GetEventRecorderFor("githubapp-controller"),
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),
Recorder: k8sManager.GetEventRecorderFor("githubapp-controller"),
HTTPClient: httpClient,
VaultClient: vaultClient,
K8sClient: k8sClientset,
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())

Expand Down
Loading

0 comments on commit 2fee4e8

Please sign in to comment.