Skip to content

Commit

Permalink
fix(RHIDP-3674, RHIDP-3965): Add MVP scenario that matches latest v1.…
Browse files Browse the repository at this point in the history
…2 version of RHDH, fix oauth2 proxy for OLM install method

Signed-off-by: Pavel Macík <[email protected]>
  • Loading branch information
pmacik committed Sep 17, 2024
1 parent 8959f01 commit 1252581
Show file tree
Hide file tree
Showing 18 changed files with 687 additions and 134 deletions.
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ export RHDH_HELM_CHART_VERSION ?=
export RHDH_HELM_RELEASE_NAME ?= rhdh

# RHDH OLM subscription to deploy
export RHDH_OLM_INDEX_IMAGE ?= quay.io/rhdh/iib:1.2-v$(shell oc version -o json | jq -r '.openshiftVersion' | sed -r -e "s,([0-9]+\.[0-9]+)\..+,\1,")-$(shell oc version -o json | jq -r '.serverVersion.platform' | sed -r -e "s,linux/,," | sed -e 's,amd64,x86_64,')
export RHDH_OLM_CHANNEL ?= fast
export RHDH_OLM_INDEX_IMAGE ?= registry.redhat.io/redhat/redhat-operator-index:v$(shell oc version -o json | jq -r '.openshiftVersion' | sed -r -e "s,([0-9]+\.[0-9]+)\..+,\1,")
export RHDH_OLM_CHANNEL ?= fast-1.2

# RHDH horizontal scaling
export RHDH_DEPLOYMENT_REPLICAS ?= 1
Expand All @@ -48,7 +48,7 @@ export RHDH_RESOURCES_MEMORY_REQUESTS ?=
export RHDH_RESOURCES_MEMORY_LIMITS ?=
export RHDH_KEYCLOAK_REPLICAS ?= 1
export LOCUST_EXTRA_CMD ?=
export AUTH_PROVIDER ?=
export AUTH_PROVIDER ?= keycloak

# RHDH install method - one of 'helm' or 'olm'
export RHDH_INSTALL_METHOD ?= helm
Expand Down Expand Up @@ -175,7 +175,7 @@ test: $(TMP_DIR) $(ARTIFACT_DIR)
ifneq ($(shell test '$(AUTH_PROVIDER)' == 'keycloak' && echo 1 || echo 0),0)
$(eval key_pass := $(shell oc -n rhdh-performance get secret perf-test-secrets -o template --template='{{.data.keycloak_user_pass}}' | base64 -d))
$(eval key_host := $(shell oc -n rhdh-performance get routes/keycloak -o template --template='{{.spec.host}}' ))
$(eval LOCUST_EXTRA_CMD := --keycloak-host $(key_host) --keycloak-password $(key_pass) )
$(eval LOCUST_EXTRA_CMD := $(LOCUST_EXTRA_CMD) --keycloak-host $(key_host) --keycloak-password $(key_pass) )
ifneq ($(shell test $(USERS) -gt $(WORKERS) && echo 1 || echo 0),0)
@echo "users greater than workers "
else
Expand Down
4 changes: 2 additions & 2 deletions ci-scripts/dev-sandbox/Makefile
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#SHELL="/bin/bash -x"

export RHDH_OLM_INDEX_IMAGE ?= quay.io/rhdh/iib:1.2-v4.15-x86_64
export RHDH_OLM_INDEX_IMAGE ?= registry.redhat.io/redhat/redhat-operator-index:v4.15
export RHDH_OPERATOR_NAMESPACE ?= rhdh-operator
export RHDH_OLM_CHANNEL ?= fast
export RHDH_OLM_CHANNEL ?= fast-1.2
export RHDH_OLM_OPERATOR_PACKAGE ?= rhdh

export ARTIFACT_DIR ?=
Expand Down
6 changes: 5 additions & 1 deletion ci-scripts/dev-sandbox/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ collect_counts "baseline-counts-post"

# testing env
if [ "$RHDH_INSTALL_METHOD" == "olm" ]; then
rhdh_route="backstage-developer-hub"
if [ "$AUTH_PROVIDER" == "keycloak" ]; then
rhdh_route="rhdh"
else
rhdh_route="backstage-developer-hub"
fi
elif [ "$RHDH_INSTALL_METHOD" == "helm" ]; then
export RHDH_HELM_RELEASE_NAME RHDH_HELM_CHART

Expand Down
73 changes: 52 additions & 21 deletions ci-scripts/rhdh-setup/create_resource.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,11 @@ backstage_url() {
if [ "$RHDH_INSTALL_METHOD" == "helm" ]; then
rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}"
else
rhdh_route="backstage-developer-hub"
if [ "$AUTH_PROVIDER" == "keycloak" ]; then
rhdh_route="rhdh"
else
rhdh_route="backstage-developer-hub"
fi
fi
echo -n "https://$(oc get routes "${rhdh_route}" -n "${RHDH_NAMESPACE}" -o jsonpath='{.spec.host}')" >"$f"
fi
Expand Down Expand Up @@ -149,14 +153,27 @@ create_cmp() {
}

create_group() {
token=$(get_token)
groupname="group${0}"
echo " g, group:default/${groupname}, role:default/perf_admin" >>"$TMP_DIR/group-rbac.yaml"
curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/groups" \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer '"$token" \
--data-raw '{"name": "'"${groupname}"'"}' |& tee -a "$TMP_DIR/create_group.log"
echo "[INFO][$(date --utc -Ins)] Group $groupname created" >>"$TMP_DIR/create_group.log"
max_attempts=5
attempt=1
while ((attempt <= max_attempts)); do
token=$(get_token)
groupname="group${0}"
echo " g, group:default/${groupname}, role:default/perf_admin" >>"$TMP_DIR/group-rbac.yaml"
curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/groups" \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer '"$token" \
--data-raw '{"name": "'"${groupname}"'"}' |& tee -a "$TMP_DIR/create_group.log"
if [ "${PIPESTATUS[0]}" -eq 0 ]; then
echo "[INFO][$(date --utc -Ins)] Group $groupname created" >>"$TMP_DIR/create_group.log"
return
else
echo "[WARNING][$(date --utc -Ins)] Unable to create the $groupname group at $attempt. attempt. Trying again up to $max_attempts times." >>"$TMP_DIR/create_group.log"
((attempt++))
fi
done
if [[ $attempt -gt $max_attempts ]]; then
echo "[ERROR][$(date --utc -Ins)] Unable to create the $groupname group in $max_attempts attempts, giving up!" |& tee -a "$TMP_DIR/create_group.log"
fi
}

create_groups() {
Expand All @@ -166,16 +183,29 @@ create_groups() {
}

create_user() {
token=$(get_token)
grp=$(echo "${0}%${GROUP_COUNT}" | bc)
[[ $grp -eq 0 ]] && grp=${GROUP_COUNT}
username="test${0}"
groupname="group${grp}"
curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/users" \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer '"$token" \
--data-raw '{"firstName":"'"${username}"'","lastName":"tester", "email":"'"${username}"'@test.com","emailVerified":"true", "enabled":"true", "username":"'"${username}"'","groups":["/'"${groupname}"'"],"credentials":[{"type":"password","value":"'"${KEYCLOAK_USER_PASS}"'","temporary":false}]}' |& tee -a "$TMP_DIR/create_user.log"
echo "[INFO][$(date --utc -Ins)] User $username ($groupname) created" >>"$TMP_DIR/create_user.log"
max_attempts=5
attempt=1
while ((attempt <= max_attempts)); do
token=$(get_token)
grp=$(echo "${0}%${GROUP_COUNT}" | bc)
[[ $grp -eq 0 ]] && grp=${GROUP_COUNT}
username="test${0}"
groupname="group${grp}"
curl -s -k --location --request POST "$(keycloak_url)/auth/admin/realms/backstage/users" \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer '"$token" \
--data-raw '{"firstName":"'"${username}"'","lastName":"tester", "email":"'"${username}"'@test.com","emailVerified":"true", "enabled":"true", "username":"'"${username}"'","groups":["/'"${groupname}"'"],"credentials":[{"type":"password","value":"'"${KEYCLOAK_USER_PASS}"'","temporary":false}]}' |& tee -a "$TMP_DIR/create_user.log"
if [ "${PIPESTATUS[0]}" -eq 0 ]; then
echo "[INFO][$(date --utc -Ins)] User $username ($groupname) created" >>"$TMP_DIR/create_user.log"
return
else
echo "[WARNING][$(date --utc -Ins)] Unable to create the $username user at $attempt. attempt. Trying again up to $max_attempts times." >>"$TMP_DIR/create_user.log"
((attempt++))
fi
done
if [[ $attempt -gt $max_attempts ]]; then
echo "[ERROR][$(date --utc -Ins)] Unable to create the $username user in $max_attempts attempts, giving up!" |& tee -a "$TMP_DIR/create_user.log"
fi
}

create_users() {
Expand Down Expand Up @@ -267,19 +297,20 @@ get_token() {
trap "rm -rf $token_lockfile; exit" INT TERM EXIT HUP

timeout_timestamp=$(date -d "60 seconds" "+%s")
while [ ! -f "$token_file" ] || [ ! -s "$token_file" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ]; do
while [ ! -f "$token_file" ] || [ ! -s "$token_file" ] || [ -z "$(jq -rc '.expires_in_timestamp' "$token_file")" ] || [ "$(date +%s)" -gt "$(jq -rc '.expires_in_timestamp' "$token_file")" ]; do
log_token_info "Refreshing keycloak token"
if [ "$(date "+%s")" -gt "$timeout_timestamp" ]; then
log_token_err "Timeout getting keycloak token"
exit 1
fi
if [[ ${service} == 'rhdh' ]]; then
log_token_info "Refreshing RHDH token"
[[ -f "$token_file" ]] && rm -rf "$token_file" && rm -rf "$TMP_DIR/cookie.jar"
if ! rhdh_token >"$token_file"; then
log_token_err "Unable to get token, re-attempting"
fi
else
keycloak_pass=$(oc -n "${RHDH_NAMESPACE}" get secret credential-example-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d)
keycloak_pass=$(oc -n "${RHDH_NAMESPACE}" get secret credential-rhdh-sso -o template --template='{{.data.ADMIN_PASSWORD}}' | base64 -d)
if ! keycloak_token >"$token_file"; then
log_token_err "Unable to get token, re-attempting"
fi
Expand Down
33 changes: 29 additions & 4 deletions ci-scripts/rhdh-setup/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ export RHDH_HELM_CHART=${RHDH_HELM_CHART:-redhat-developer-hub}
export RHDH_HELM_CHART_VERSION=${RHDH_HELM_CHART_VERSION:-}

OCP_VER="$(oc version -o json | jq -r '.openshiftVersion' | sed -r -e "s#([0-9]+\.[0-9]+)\..+#\1#")"
OCP_ARCH="$(oc version -o json | jq -r '.serverVersion.platform' | sed -r -e "s#linux/##" | sed -e 's#amd64#x86_64#')"
export RHDH_OLM_INDEX_IMAGE="${RHDH_OLM_INDEX_IMAGE:-quay.io/rhdh/iib:1.2-v${OCP_VER}-${OCP_ARCH}}"
export RHDH_OLM_CHANNEL=${RHDH_OLM_CHANNEL:-fast}
#OCP_ARCH="$(oc version -o json | jq -r '.serverVersion.platform' | sed -r -e "s#linux/##" | sed -e 's#amd64#x86_64#')"
export RHDH_OLM_INDEX_IMAGE="${RHDH_OLM_INDEX_IMAGE:-registry.redhat.io/redhat/redhat-operator-index:v${OCP_VER}}"
export RHDH_OLM_CHANNEL=${RHDH_OLM_CHANNEL:-fast-1.2}
export RHDH_OLM_OPERATOR_PACKAGE=${RHDH_OLM_OPERATOR_PACKAGE:-rhdh}

export PRE_LOAD_DB="${PRE_LOAD_DB:-true}"
Expand Down Expand Up @@ -173,6 +173,15 @@ keycloak_install() {
envsubst <template/keycloak/keycloak.yaml | $clin apply -f -
wait_to_start statefulset keycloak 450 600
envsubst <template/keycloak/keycloakRealm.yaml | $clin apply -f -
if [ "$INSTALL_METHOD" == "helm" ]; then
export OAUTH2_REDIRECT_URI=https://${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}/oauth2/callback
elif [ "$INSTALL_METHOD" == "olm" ]; then
if [ "$AUTH_PROVIDER" == "keycloak" ]; then
export OAUTH2_REDIRECT_URI=https://rhdh-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}/oauth2/callback
else
export OAUTH2_REDIRECT_URI=https://backstage-developer-hub-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}/oauth2/callback
fi
fi
envsubst <template/keycloak/keycloakClient.yaml | $clin apply -f -
envsubst <template/keycloak/keycloakUser.yaml | $clin apply -f -
}
Expand All @@ -197,6 +206,18 @@ backstage_install() {
cp "template/backstage/app-config.yaml" "$TMP_DIR/app-config.yaml"
if [ "${AUTH_PROVIDER}" == "keycloak" ]; then yq -i '. |= . + {"signInPage":"oauth2Proxy"}' "$TMP_DIR/app-config.yaml"; fi
if [ "${AUTH_PROVIDER}" == "keycloak" ]; then yq -i '. |= . + {"auth":{"environment":"production","providers":{"oauth2Proxy":{}}}}' "$TMP_DIR/app-config.yaml"; else yq -i '. |= . + {"auth":{"providers":{"guest":{"dangerouslyAllowOutsideDevelopment":true}}}}' "$TMP_DIR/app-config.yaml"; fi
if [ "$INSTALL_METHOD" == "helm" ]; then
base_url="https://${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}"
elif [ "$INSTALL_METHOD" == "olm" ]; then
if [ "$AUTH_PROVIDER" == "keycloak" ]; then
base_url="https://rhdh-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}"
else
base_url="https://backstage-developer-hub-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}"
fi
fi
yq -i '.app.baseUrl="'"$base_url"'"' "$TMP_DIR/app-config.yaml"
yq -i '.backend.baseUrl="'"$base_url"'"' "$TMP_DIR/app-config.yaml"
yq -i '.backend.cors.origin="'"$base_url"'"' "$TMP_DIR/app-config.yaml"
until envsubst <template/backstage/secret-rhdh-pull-secret.yaml | $clin apply -f -; do $clin delete secret rhdh-pull-secret --ignore-not-found=true; done
if ${ENABLE_RBAC}; then yq -i '. |= . + load("template/backstage/'$INSTALL_METHOD'/app-rbac-patch.yaml")' "$TMP_DIR/app-config.yaml"; fi
until $clin create configmap app-config-rhdh --from-file "app-config.rhdh.yaml=$TMP_DIR/app-config.yaml"; do $clin delete configmap app-config-rhdh --ignore-not-found=true; done
Expand All @@ -215,7 +236,7 @@ backstage_install() {
return 1
fi
if [ "${AUTH_PROVIDER}" == "keycloak" ] && ${RHDH_METRIC}; then $clin create -f template/backstage/rhdh-metrics-service.yaml; fi
if ${RHDH_METRIC}; then envsubst <template/backstage/rhdh-servicemonitor.yaml| $clin create -f -; fi
if ${RHDH_METRIC}; then envsubst <template/backstage/rhdh-servicemonitor.yaml | $clin create -f -; fi
}

# shellcheck disable=SC2016,SC1004
Expand Down Expand Up @@ -292,6 +313,10 @@ install_rhdh_with_olm() {
set +x
wait_for_crd backstages.rhdh.redhat.com

if [ "$AUTH_PROVIDER" == "keycloak" ]; then
envsubst <template/backstage/olm/rhdh-oauth2.deployment.yaml | $clin apply -f -
fi

backstage_yaml="$TMP_DIR/backstage.yaml"
envsubst <template/backstage/olm/backstage.yaml >"$backstage_yaml"
if ${ENABLE_RBAC}; then
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: rhdh
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/component: oauth2
app.kubernetes.io/instance: rhdh
app.kubernetes.io/name: developer-hub
template:
metadata:
labels:
app.kubernetes.io/component: oauth2
app.kubernetes.io/instance: rhdh
app.kubernetes.io/name: developer-hub
spec:
containers:
- args:
- --provider=oidc
- --email-domain=*
- --upstream=http://backstage-developer-hub:80
- --http-address=0.0.0.0:4180
- --skip-provider-button
env:
- name: OAUTH2_PROXY_CLIENT_ID
valueFrom:
secretKeyRef:
key: CLIENT_ID
name: keycloak-client-secret-backstage
- name: OAUTH2_PROXY_CLIENT_SECRET
valueFrom:
secretKeyRef:
key: CLIENT_SECRET
name: keycloak-client-secret-backstage
- name: OAUTH2_PROXY_COOKIE_SECRET
valueFrom:
secretKeyRef:
key: keycloak_cookie_secret
name: perf-test-secrets
- name: OAUTH2_PROXY_OIDC_ISSUER_URL
value: https://keycloak-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}/auth/realms/backstage
- name: OAUTH2_PROXY_SSL_INSECURE_SKIP_VERIFY
value: "true"
image: quay.io/oauth2-proxy/oauth2-proxy:latest
imagePullPolicy: IfNotPresent
name: oauth2-proxy
ports:
- containerPort: 4180
name: oauth2-proxy
protocol: TCP
resources: {}
imagePullSecrets:
- name: rhdh-pull-secret
---
apiVersion: v1
kind: Service
metadata:
name: rhdh
spec:
ports:
- name: oauth2-proxy
port: 4180
protocol: TCP
targetPort: oauth2-proxy
selector:
app.kubernetes.io/component: oauth2
app.kubernetes.io/instance: rhdh
app.kubernetes.io/name: developer-hub
type: ClusterIP
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
name: rhdh
spec:
path: /
port:
targetPort: 4180
tls:
insecureEdgeTerminationPolicy: Redirect
termination: edge
to:
kind: Service
name: rhdh
weight: 100
wildcardPolicy: None
2 changes: 1 addition & 1 deletion ci-scripts/rhdh-setup/template/keycloak/keycloak.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: keycloak.org/v1alpha1
kind: Keycloak
metadata:
name: example-sso
name: rhdh-sso
labels:
app: sso
spec:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ spec:
implicitFlowEnabled: false
publicClient: false
redirectUris:
- https://${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}-${RHDH_NAMESPACE}.${OPENSHIFT_APP_DOMAIN}/oauth2/callback
- ${OAUTH2_REDIRECT_URI}
serviceAccountsEnabled: true
standardFlowEnabled: true
realmSelector:
Expand Down
8 changes: 5 additions & 3 deletions ci-scripts/scalability/test-scalability.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,14 @@ wait_for_indexing() {
COOKIE="$TMP_DIR/cookie.jar"
if [ "$INSTALL_METHOD" == "helm" ]; then
rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}"
#rhdh_route="rhdh-redhat-developer-hub"
else
rhdh_route="backstage-developer-hub"
if [ "$AUTH_PROVIDER" == "keycloak" ]; then
rhdh_route="rhdh"
else
rhdh_route="backstage-developer-hub"
fi
fi
if [ "$WAIT_FOR_SEARCH_INDEX" == "true" ]; then
rhdh_route="${RHDH_HELM_RELEASE_NAME}-${RHDH_HELM_CHART}"
HOST="https://$(oc get routes "${rhdh_route}" -n "${RHDH_NAMESPACE:-rhdh-performance}" -o jsonpath='{.spec.host}')"

start=$(date +%s)
Expand Down
6 changes: 3 additions & 3 deletions ci-scripts/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ GITHUB_USER=$(cat /usr/local/ci-secrets/backstage-performance/github.user)
GITHUB_REPO=$(cat /usr/local/ci-secrets/backstage-performance/github.repo)
QUAY_TOKEN=$(cat /usr/local/ci-secrets/backstage-performance/quay.token)

export RHDH_DEPLOYMENT_REPLICAS=${RHDH_DEPLOYMENT_REPLICAS:-5}
export RHDH_DB_REPLICAS=${RHDH_DB_REPLICAS:-5}
export RHDH_DEPLOYMENT_REPLICAS=${RHDH_DEPLOYMENT_REPLICAS:-1}
export RHDH_DB_REPLICAS=${RHDH_DB_REPLICAS:-1}
export RHDH_DB_STORAGE=${RHDH_DB_STORAGE:-1Gi}
export RHDH_KEYCLOAK_REPLICAS=${RHDH_KEYCLOAK_REPLICAS:-5}
export RHDH_KEYCLOAK_REPLICAS=${RHDH_KEYCLOAK_REPLICAS:-1}

export API_COUNT=${API_COUNT:-1000}
export COMPONENT_COUNT=${COMPONENT_COUNT:-1000}
Expand Down
8 changes: 6 additions & 2 deletions ci-scripts/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,15 @@ source "$(readlink -m "$SCRIPT_DIR"/../test.env)"
export SCENARIO RHDH_INSTALL_METHOD AUTH_PROVIDER

RHDH_INSTALL_METHOD=${RHDH_INSTALL_METHOD:-helm}
AUTH_PROVIDER=${AUTH_PROVIDER:-}
AUTH_PROVIDER=${AUTH_PROVIDER:-keycloak}

# testing env
if [ "$RHDH_INSTALL_METHOD" == "olm" ]; then
rhdh_route="backstage-developer-hub"
if [ "$AUTH_PROVIDER" == "keycloak" ]; then
rhdh_route="rhdh"
else
rhdh_route="backstage-developer-hub"
fi
elif [ "$RHDH_INSTALL_METHOD" == "helm" ]; then
export RHDH_HELM_RELEASE_NAME RHDH_HELM_CHART

Expand Down
Loading

0 comments on commit 1252581

Please sign in to comment.