Merge branch 'master' of github.com:flyteorg/flyte into update-k8s-dips

.github/workflows/checks.yml

@@ -1,5 +1,9 @@
 name: Components Checks
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
     paths:

.github/workflows/codespell.yml

@@ -1,6 +1,10 @@
 ---
 name: Codespell
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches: [master]

.github/workflows/flyteidl-checks.yml

@@ -1,5 +1,9 @@
 name: Flyteidl Verification Tests
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
     paths:

.github/workflows/helm-charts.yaml

@@ -1,5 +1,9 @@
 name: Package & Push Flyte Helm Charts
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
   push:

.github/workflows/sandbox.yml

@@ -1,5 +1,9 @@
 name: Build & Push Sandbox Docker Image
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
     paths:

.github/workflows/single-binary.yml

@@ -1,5 +1,9 @@
 name: Build & Push Flyte Single Binary Images
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
     paths:

.github/workflows/tests.yml

@@ -1,4 +1,9 @@
 name: tests
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches:

datacatalog/.github/workflows/checks.yml

@@ -1,5 +1,9 @@
 name: Datacatalog Checks
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   pull_request:

docker/sandbox-bundled/kustomize/complete-agent/kustomization.yaml

@@ -10,3 +10,6 @@ helmCharts:
 namespace: flyte
 resources:
 - ../namespace.yaml
+
+patchesStrategicMerge:
+  - patch.yaml

docker/sandbox-bundled/kustomize/complete-agent/patch.yaml

@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flyteagent
+  namespace: flyte
+spec:
+  template:
+    spec:
+      containers:
+      - name: flyteagent
+        env:
+        - name: FLYTE_AWS_ENDPOINT
+          value: http://flyte-sandbox-minio.flyte:9000
+        - name: FLYTE_AWS_ACCESS_KEY_ID
+          value: minio
+        - name: FLYTE_AWS_SECRET_ACCESS_KEY
+          value: miniostorage

docker/sandbox-bundled/manifests/complete-agent.yaml

@@ -816,7 +816,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: R2JRWFVRYThnRFVLbHpuSA==
+  haSharedSecret: NlhtNUl5amRScVFNVHVPRQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1409,7 +1409,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 0ee1553aec7c03152a0a44e7b1a82985795774412a779f7b607a57e59f42c8ef
+        checksum/secret: 4816300df83f72e9be3652c291978fb2e05b80b8afcc3dfd6812f4aa28d5f640
       labels:
         app: docker-registry
         release: flyte-sandbox
@@ -1727,7 +1727,6 @@ spec:
       app.kubernetes.io/name: flyteagent
   template:
     metadata:
-      annotations: null
       labels:
         app.kubernetes.io/instance: flyteagent
         app.kubernetes.io/managed-by: Helm
@@ -1738,6 +1737,13 @@ spec:
       - command:
         - pyflyte
         - serve
+        env:
+        - name: FLYTE_AWS_ENDPOINT
+          value: http://flyte-sandbox-minio.flyte:9000
+        - name: FLYTE_AWS_ACCESS_KEY_ID
+          value: minio
+        - name: FLYTE_AWS_SECRET_ACCESS_KEY
+          value: miniostorage
         image: ghcr.io/flyteorg/flyteagent:1.9.1
         imagePullPolicy: IfNotPresent
         name: flyteagent

docker/sandbox-bundled/manifests/complete.yaml

@@ -805,7 +805,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: d1l6eWRCOXBJcFhiNEo5QQ==
+  haSharedSecret: dVREVFFlOUx5dWtyRzNhNg==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1366,7 +1366,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 2f5b6d46fd3276b5b25c8a537298beb6943b13b0b21900db8b2da23e166f0593
+        checksum/secret: b96da081475b0e9dc4818925c2b987d4a03f4ae8cdd13c5646882144c377d80f
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/dev.yaml

@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: UkFsUVRMRndZeTNJUVNFSA==
+  haSharedSecret: dng0dkhOTDZTYzNVWXQweQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -933,7 +933,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 25a046ef1aaf34ffb59f7b92554e1cfd0015b9a11f7f165ce06bba31e3bced1b
+        checksum/secret: bc60a1502c30ce6508ac0b78e0b15a13e624a5671ca81548f1782cb21e540d58
       labels:
         app: docker-registry
         release: flyte-sandbox

flyteadmin/.github/workflows/checks.yml

@@ -1,5 +1,9 @@
 name: Flyteadmin Checks
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   pull_request:

flyteadmin/auth/authzserver/metadata_provider.go

@@ -2,14 +2,22 @@ package authzserver
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
+
+	"google.golang.org/grpc/codes"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
 
 	"github.com/flyteorg/flyte/flyteadmin/auth"
 	authConfig "github.com/flyteorg/flyte/flyteadmin/auth/config"
+	flyteErrors "github.com/flyteorg/flyte/flyteadmin/pkg/errors"
 	"github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service"
+	"github.com/flyteorg/flyte/flytestdlib/logger"
 )
 
 type OAuth2MetadataProvider struct {
@@ -72,7 +80,7 @@ func (s OAuth2MetadataProvider) GetOAuth2Metadata(ctx context.Context, r *servic
 			httpClient.Transport = transport
 		}
 
-		response, err := httpClient.Get(externalMetadataURL.String())
+		response, err := sendAndRetryHTTPRequest(ctx, httpClient, externalMetadataURL.String(), s.cfg.AppAuth.ExternalAuthServer.RetryAttempts, s.cfg.AppAuth.ExternalAuthServer.RetryDelay.Duration)
 		if err != nil {
 			return nil, err
 		}
@@ -107,3 +115,41 @@ func NewService(config *authConfig.Config) OAuth2MetadataProvider {
 		cfg: config,
 	}
 }
+
+func sendAndRetryHTTPRequest(ctx context.Context, client *http.Client, url string, retryAttempts int, retryDelay time.Duration) (*http.Response, error) {
+	var response *http.Response
+	var err error
+	totalAttempts := retryAttempts + 1 // Add one for initial http request attempt
+
+	backoff := wait.Backoff{
+		Duration: retryDelay,
+		Steps:    totalAttempts,
+	}
+
+	retryableOauthMetadataError := flyteErrors.NewFlyteAdminError(codes.Internal, "Failed to get oauth metadata.")
+	err = retry.OnError(backoff,
+		func(err error) bool { // Determine if error is retryable
+			return err == retryableOauthMetadataError
+		}, func() error { // Send HTTP request
+			response, err = client.Get(url)
+			if err != nil {
+				logger.Errorf(ctx, "Failed to send oauth metadata HTTP request. Err: %v", err)
+				return err
+			}
+			if response != nil && response.StatusCode >= http.StatusUnauthorized && response.StatusCode <= http.StatusNetworkAuthenticationRequired {
+				logger.Errorf(ctx, "Failed to get oauth metadata, going to retry. StatusCode: %v Err: %v", response.StatusCode, err)
+				return retryableOauthMetadataError
+			}
+			return nil
+		})
+
+	if err != nil {
+		return nil, err
+	}
+
+	if response != nil && response.StatusCode != http.StatusOK {
+		return response, fmt.Errorf("failed to get oauth metadata with status code %v", response.StatusCode)
+	}
+
+	return response, nil
+}

flyteadmin/auth/authzserver/metadata_provider_test.go

@@ -16,6 +16,8 @@ import (
 	config2 "github.com/flyteorg/flyte/flytestdlib/config"
 )
 
+var oauthMetadataFailureErrorMessage = "Failed to get oauth metadata."
+
 func TestOAuth2MetadataProvider_FlyteClient(t *testing.T) {
 	provider := NewService(&authConfig.Config{
 		AppAuth: authConfig.OAuth2Options{
@@ -111,3 +113,84 @@ func TestOAuth2MetadataProvider_OAuth2Metadata(t *testing.T) {
 		assert.Equal(t, "https://dev-14186422.okta.com", resp.Issuer)
 	})
 }
+
+func TestSendAndRetryHttpRequest(t *testing.T) {
+	t.Run("Retry into failure", func(t *testing.T) {
+		requestAttempts := 0
+		hf := func(w http.ResponseWriter, r *http.Request) {
+			switch strings.TrimSpace(r.URL.Path) {
+			case "/":
+				w.WriteHeader(500)
+				requestAttempts++
+			default:
+				http.NotFoundHandler().ServeHTTP(w, r)
+			}
+		}
+
+		server := httptest.NewServer(http.HandlerFunc(hf))
+		defer server.Close()
+		http.DefaultClient = server.Client()
+		retryAttempts := 5
+		totalAttempts := retryAttempts + 1 // 1 for the initial try
+
+		resp, err := sendAndRetryHTTPRequest(context.Background(), server.Client(), server.URL, retryAttempts, 0 /* for testing */)
+		assert.Error(t, err)
+		assert.Equal(t, oauthMetadataFailureErrorMessage, err.Error())
+		assert.Nil(t, resp)
+		assert.Equal(t, totalAttempts, requestAttempts)
+	})
+
+	t.Run("Retry into success", func(t *testing.T) {
+		requestAttempts := 0
+		hf := func(w http.ResponseWriter, r *http.Request) {
+			switch strings.TrimSpace(r.URL.Path) {
+			case "/":
+				if requestAttempts > 2 {
+					w.WriteHeader(200)
+				} else {
+					requestAttempts++
+					w.WriteHeader(500)
+				}
+			default:
+				http.NotFoundHandler().ServeHTTP(w, r)
+			}
+		}
+
+		server := httptest.NewServer(http.HandlerFunc(hf))
+		defer server.Close()
+		http.DefaultClient = server.Client()
+		retryAttempts := 5
+		expectedRequestAttempts := 3
+
+		resp, err := sendAndRetryHTTPRequest(context.Background(), server.Client(), server.URL, retryAttempts, 0 /* for testing */)
+		assert.NoError(t, err)
+		assert.NotNil(t, resp)
+		assert.Equal(t, 200, resp.StatusCode)
+		assert.Equal(t, expectedRequestAttempts, requestAttempts)
+	})
+
+	t.Run("Success", func(t *testing.T) {
+		requestAttempts := 0
+		hf := func(w http.ResponseWriter, r *http.Request) {
+			switch strings.TrimSpace(r.URL.Path) {
+			case "/":
+				w.WriteHeader(200)
+			default:
+				http.NotFoundHandler().ServeHTTP(w, r)
+			}
+		}
+
+		server := httptest.NewServer(http.HandlerFunc(hf))
+		defer server.Close()
+		http.DefaultClient = server.Client()
+		retryAttempts := 5
+		expectedRequestAttempts := 0
+
+		resp, err := sendAndRetryHTTPRequest(context.Background(), server.Client(), server.URL, retryAttempts, 0 /* for testing */)
+		assert.NoError(t, err)
+		assert.NotNil(t, resp)
+		assert.Equal(t, 200, resp.StatusCode)
+		assert.Equal(t, expectedRequestAttempts, requestAttempts)
+	})
+
+}

flyteadmin/auth/config/config.go

@@ -79,6 +79,10 @@ var (
 			},
 		},
 		AppAuth: OAuth2Options{
+			ExternalAuthServer: ExternalAuthorizationServer{
+				RetryAttempts: 5,
+				RetryDelay:    config.Duration{Duration: 1 * time.Second},
+			},
 			AuthServerType: AuthorizationServerTypeSelf,
 			ThirdParty: ThirdPartyConfigOptions{
 				FlyteClientConfig: FlyteClientConfig{
@@ -191,7 +195,9 @@ type ExternalAuthorizationServer struct {
 	AllowedAudience     []string   `json:"allowedAudience" pflag:",Optional: A list of allowed audiences. If not provided, the audience is expected to be the public Uri of the service."`
 	MetadataEndpointURL config.URL `json:"metadataUrl" pflag:",Optional: If the server doesn't support /.well-known/oauth-authorization-server, you can set a custom metadata url here.'"`
 	// HTTPProxyURL allows operators to access external OAuth2 servers using an external HTTP Proxy
-	HTTPProxyURL config.URL `json:"httpProxyURL" pflag:",OPTIONAL: HTTP Proxy to be used for OAuth requests."`
+	HTTPProxyURL  config.URL      `json:"httpProxyURL" pflag:",OPTIONAL: HTTP Proxy to be used for OAuth requests."`
+	RetryAttempts int             `json:"retryAttempts" pflag:", Optional: The number of attempted retries on a transient failure to get the OAuth metadata"`
+	RetryDelay    config.Duration `json:"retryDelay" pflag:", Optional, Duration to wait between retries"`
 }
 
 // OAuth2Options defines settings for app auth.