WIP: Openshift 3 Rebase PR

api/v1alpha1/airflowbase_types.go

@@ -26,15 +26,12 @@ import (
 
 // defaults and constant strings
 const (
-	DefaultMySQLImage      = "mysql"
-	DefaultMySQLVersion    = "5.7"
-	DefaultPostgresImage   = "postgres"
-	DefaultPostgresVersion = "10"
-	defaultUIImage         = "gcr.io/airflow-operator/airflow"
-	defaultUIVersion       = "1.10.2"
-	defaultFlowerVersion   = "1.10.2"
-	defaultNFSVersion      = "0.8"
+	DefaultMySQLImage      = "registry.redhat.io/rhel8/mysql-80"
+	DefaultMySQLVersion    = "latest"
+	DefaultPostgresImage   = "registry.redhat.io/rhel8/postgresql-12"
+	DefaultPostgresVersion = "latest"
 	defaultNFSImage        = "k8s.gcr.io/volume-nfs"
+	defaultNFSVersion      = "0.8"
 	defaultSQLProxyImage   = "gcr.io/cloud-airflow-public/airflow-sqlproxy"
 	defaultSQLProxyVersion = "1.8.0"
 	defaultSchedule        = "0 0 0 ? * * *`" // daily@midnight
@@ -279,31 +276,6 @@ func validStorageProvider(provider string) bool {
 	return false
 }
 
-// AirflowUISpec defines the attributes to deploy Airflow UI component
-type AirflowUISpec struct {
-	// Image defines the AirflowUI Docker image.
-	// +optional
-	Image string `json:"image,omitempty"`
-	// Version defines the AirflowUI Docker image version.
-	// +optional
-	Version string `json:"version,omitempty"`
-	// Replicas defines the number of running Airflow UI instances in a cluster
-	// +optional
-	Replicas int32 `json:"replicas,omitempty"`
-	// Resources is the resource requests and limits for the pods.
-	// +optional
-	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
-	// enableroutes: true enables routes for the AirflowUI and CeleryUI
-	// +optional, enable on OpenShift clusters only
-	EnableRoutes bool `json:"enableroutes,omitempty"`
-}
-
-func (s *AirflowUISpec) validate(fp *field.Path) field.ErrorList {
-	errs := field.ErrorList{}
-	//errs = append(errs, s.Resources.validate(fp.Child("resources"))...)
-	return errs
-}
-
 // NFSStoreSpec defines the attributes to deploy Airflow Storage component
 type NFSStoreSpec struct {
 	// Image defines the NFS Docker image.

api/v1alpha1/airflowcluster_types.go

@@ -33,11 +33,16 @@ const (
 	defaultRedisImage       = "redis"
 	defaultRedisVersion     = "4.0"
 	defaultRedisPort        = "6379"
-	defaultWorkerImage      = "gcr.io/airflow-operator/airflow"
-	defaultSchedulerImage   = "gcr.io/airflow-operator/airflow"
-	defaultFlowerImage      = "gcr.io/airflow-operator/airflow"
 	GitsyncImage            = "k8s.gcr.io/git-sync/git-sync"
 	GitsyncVersion          = "v3.2.2"
+	defaultFlowerImage      = "quay.io/opendatahub/docker-airflow"
+	defaultFlowerVersion    = "1.10.12-1"
+	defaultSchedulerImage   = "quay.io/opendatahub/docker-airflow"
+	defaultSchedulerVersion = "1.10.12-1"
+	defaultUIImage          = "quay.io/opendatahub/docker-airflow"
+	defaultUIVersion        = "1.10.12-1"
+	defaultWorkerImage      = "quay.io/opendatahub/docker-airflow"
+	defaultWorkerVersion    = "1.10.12-1"
 	GCSsyncImage            = "gcr.io/cloud-airflow-releaser/gcs-syncd"
 	GCSsyncVersion          = "cloud_composer_service_2018-05-23-RC0"
 	ExecutorLocal           = "Local"
@@ -46,8 +51,6 @@ const (
 	ExecutorK8s             = "Kubernetes"
 	defaultExecutor         = ExecutorLocal
 	defaultBranch           = "master"
-	defaultWorkerVersion    = "1.10.2"
-	defaultSchedulerVersion = "1.10.2"
 	defaultWorkerForceRoot  = "false"
 )
 
@@ -189,9 +192,9 @@ type FlowerSpec struct {
 	// Resources is the resource requests and limits for the pods.
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
-	// enableroutes: true enables routes for the AirflowUI and CeleryUI
-	// +optional, enable on OpenShift clusters only
-	EnableRoutes bool `json:"enableroutes,omitempty"`
+	// EnableRoutes exposes Flower via OpenShit route.
+	// +optional
+	EnableRoutes bool `json:"enableRoutes,omitempty"`
 }
 
 func (s *FlowerSpec) validate(fp *field.Path) field.ErrorList {
@@ -221,6 +224,50 @@ func (s *SchedulerSpec) validate(fp *field.Path) field.ErrorList {
 	return field.ErrorList{}
 }
 
+// AirflowUISpec defines the attributes to deploy Airflow UI component
+type AirflowUISpec struct {
+	// Image defines the AirflowUI Docker image.
+	// +optional
+	Image string `json:"image,omitempty"`
+	// Version defines the AirflowUI Docker image version.
+	// +optional
+	Version string `json:"version,omitempty"`
+	// Replicas defines the number of running Airflow UI instances in a cluster
+	// +optional
+	Replicas int32 `json:"replicas,omitempty"`
+	// Resources is the resource requests and limits for the pods.
+	// +optional
+	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	// EnableRoutes exposes the Airflow UI via OpenShit route.
+	// +optional
+	EnableRoutes bool `json:"enableRoutes,omitempty"`
+	// Authentication defines the user authentication for Airflow UI.
+	// +optional
+	Authentication *AirflowUIAuthentication `json:"authentication,omitempty"`
+}
+
+func (s *AirflowUISpec) validate(fp *field.Path) field.ErrorList {
+	errs := field.ErrorList{}
+	//errs = append(errs, s.Resources.validate(fp.Child("resources"))...)
+	if s == nil {
+		return errs
+	}
+	ui := field.NewPath("spec", "ui")
+	if s.Authentication != nil {
+		var allowedAuthTypes = []string{"none", "database", "openshift"}
+		allowed := false
+		for _, authType := range allowedAuthTypes {
+			if authType == s.Authentication.Type {
+				allowed = true
+			}
+		}
+		if !allowed {
+			errs = append(errs, field.NotSupported(ui.Child("authentication", "type"), s.Authentication.Type, allowedAuthTypes))
+		}
+	}
+	return errs
+}
+
 // WorkerSpec defines the attributes and desired state of Airflow workers
 type WorkerSpec struct {
 	// Image defines the Airflow worker Docker image.
@@ -432,6 +479,19 @@ type MemoryStoreStatus struct {
 	status.ComponentMeta `json:",inline"`
 }
 
+// AirflowUIAuthentication defines the user authentication for Airflow UI
+type AirflowUIAuthentication struct {
+	// Choose the authentication type. Available types are: none, database, openshift
+	// (none = no login required, database = users stored in the database can log in,
+	// openshift = users with valid OpenShift credentials can log in).
+	Type string `json:"type,omitempty"`
+	// Select the Airflow role that is assigned to the user when first registered in the
+	// database. This setting directly translates to the AUTH_USER_REGISTRATION_ROLE setting
+	// in webserver_config.py. Airflow ships with a set of roles by default: Admin, User, Op,
+	// Viewer, Public.
+	UserRegistrationRole string `json:"userRegistrationRole,omitempty"`
+}
+
 // AirflowClusterStatus defines the observed state of AirflowCluster
 type AirflowClusterStatus struct {
 	status.Meta          `json:",inline"`

config/crd/bases/airflow.apache.org_airflowclusters.yaml

@@ -946,9 +946,8 @@ spec:
             flower:
               description: Spec for Flower component.
               properties:
-                enableroutes:
-                  description: 'enableroutes: true enables routes for the AirflowUI
-                    and CeleryUI'
+                enableRoutes:
+                  description: EnableRoutes exposes Flower via OpenShit route.
                   type: boolean
                 image:
                   description: Image defines the Flower Docker image.
@@ -1455,9 +1454,26 @@ spec:
             ui:
               description: Spec for Airflow UI component.
               properties:
-                enableroutes:
-                  description: 'enableroutes: true enables routes for the AirflowUI
-                    and CeleryUI'
+                authentication:
+                  description: Authentication defines the user authentication for
+                    Airflow UI.
+                  properties:
+                    type:
+                      description: 'Choose the authentication type. Available types
+                        are: none, database, openshift (none = no login required,
+                        database = users stored in the database can log in, openshift
+                        = users with valid OpenShift credentials can log in).'
+                      type: string
+                    userRegistrationRole:
+                      description: 'Select the Airflow role that is assigned to the
+                        user when first registered in the database. This setting directly
+                        translates to the AUTH_USER_REGISTRATION_ROLE setting in webserver_config.py.
+                        Airflow ships with a set of roles by default: Admin, User,
+                        Op, Viewer, Public.'
+                      type: string
+                  type: object
+                enableRoutes:
+                  description: EnableRoutes exposes the Airflow UI via OpenShit route.
                   type: boolean
                 image:
                   description: Image defines the AirflowUI Docker image.

config/manager/kustomization.yaml

@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 resources:
 - manager.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1

config/manager/manager.yaml

@@ -23,18 +23,17 @@ spec:
         control-plane: controller-manager
     spec:
       containers:
-      - command:
-        - /manager
-        args:
-        - --enable-leader-election
-        image: controller:latest
-        name: manager
-        imagePullPolicy: Always
-        resources:
-          limits:
-            cpu: 100m
-            memory: 300Mi
-          requests:
-            cpu: 100m
-            memory: 20Mi
+        - command:
+            - /manager
+          args:
+            - --enable-leader-election
+          image: controller:latest
+          name: manager
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+            requests:
+              cpu: 100m
+              memory: 300Mi
       terminationGracePeriodSeconds: 10

config/rbac/role.yaml

@@ -18,6 +18,32 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/log
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:
@@ -67,6 +93,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - airflow.apache.org
+  resources:
+  - airflowbases/finalizers
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - airflow.apache.org
   resources:
@@ -88,6 +122,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - airflow.apache.org
+  resources:
+  - airflowclusters/finalizers
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - airflow.apache.org
   resources:
@@ -144,6 +186,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - route.openshift.io
   resources:

controllers/airflowbase_controller.go

@@ -33,6 +33,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"encoding/base64"
+	"time"
+
 	alpha1 "github.com/apache/airflow-on-k8s-operator/api/v1alpha1"
 	"github.com/apache/airflow-on-k8s-operator/controllers/application"
 	"github.com/apache/airflow-on-k8s-operator/controllers/common"
@@ -44,7 +46,6 @@ import (
 	"sigs.k8s.io/controller-reconciler/pkg/reconciler"
 	"sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/k8s"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	"time"
 )
 
 // AirflowBaseReconciler reconciles a AirflowBase object
@@ -54,7 +55,8 @@ type AirflowBaseReconciler struct {
 	Scheme *runtime.Scheme
 }
 
-// +kubebuilder:rbac:groups=airflow.apache.org,resources=airflowbases;airflowbases/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=airflow.apache.org,resources=airflowbases,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=airflow.apache.org,resources=airflowbases/finalizers,verbs=get;update;patch
 // +kubebuilder:rbac:groups=airflow.apache.org,resources=airflowbases/status,verbs=get;update;patch
 
 // Reconcile - Dummy TODO remove this