diff --git a/UPGRADE-3.0.md b/UPGRADE-3.0.md index 30eaebfd5..1b42a7868 100644 --- a/UPGRADE-3.0.md +++ b/UPGRADE-3.0.md @@ -3,8 +3,9 @@ ## Upgrade docker compose stack ```bash -bin/setup.sh \ - && dc run --rm configurator bin/console migration:20230807 +bin/migrate.sh \ + && bin/setup.sh \ + && dc run --rm configurator migration:v20230807 -vvv ``` ## Upgrade HELM release @@ -12,14 +13,12 @@ bin/setup.sh \ Upgrade helm release then run the following job: ```bash -export MIGRATION_NAME=20230807 -helm -n get values -o yaml > /tmp/.current-values.yaml \ - && helm template -f /tmp/.current-values.yaml \ +export MIGRATION_NAME=v20230807 +export RELEASE_NAME= +helm -n ${RELEASE_NAME} get values -o yaml > /tmp/.current-values.yaml \ + && helm template ${RELEASE_NAME} -f /tmp/.current-values.yaml \ --set "configurator.executeMigration=${MIGRATION_NAME}" \ -s templates/job-tests.yaml | kubectl apply -f - kubectl attach -it pod/${MIGRATION_NAME} kubectl delete -it pod/${MIGRATION_NAME} ``` - - -# TODO Link Accounts: Attach migrated users to newly configured IdP diff --git a/bin/dev/migrate-to-keycloak.sh b/bin/dev/migrate-to-keycloak.sh index 7a408095d..ef896fc12 100755 --- a/bin/dev/migrate-to-keycloak.sh +++ b/bin/dev/migrate-to-keycloak.sh @@ -2,10 +2,8 @@ set -e -NS=${NS:-"ps"} -RELEASE_NAME=${4:-"ps"} - -kubectl config use-context minikube +NS=${NS:-"ps-test"} +RELEASE_NAME=${1:-"ps"} source ./tmp/.helm.env @@ -24,26 +22,12 @@ if [ -z "${NEW_CHART_VALUES}" ]; then exit 1 fi -MIGRATION_NAME=20230807 - - -# -(cd ${NEW_CHART_DIR} \ - && helm -n ${NS} get values ${RELEASE_NAME} -o yaml > /tmp/.current-values.yaml \ - && helm -n ${NS} upgrade ${RELEASE_NAME} ./ \ - -f ${NEW_CHART_VALUES} \ - -f /tmp/.current-values.yaml \ - --set "configurator.executeMigration=${MIGRATION_NAME}" \ - --set "stack.runMigrations=false" \ -) - -exit -# - - +MIGRATION_NAME="v20230807" read -p "Reset? (y/N)" RESET_RELEASE +kubectl config use-context ${K8S_CONTEXT:-"minikube"} + if [[ $RESET_RELEASE == "y" ]]; then echo "Resetting release..." if [ -z "${OLD_CHART_VALUES}" ]; then @@ -73,13 +57,16 @@ echo "Migrating..." (cd ${NEW_CHART_DIR} \ && helm -n ${NS} get values ${RELEASE_NAME} -o yaml > /tmp/.current-values.yaml \ && helm -n ${NS} upgrade ${RELEASE_NAME} ./ \ + -f /tmp/.current-values.yaml \ -f ${NEW_CHART_VALUES} \ + && echo "Executing migrations..." \ && helm -n ${NS} upgrade ${RELEASE_NAME} ./ \ - -f ${NEW_CHART_VALUES} \ - -f /tmp/.current-values.yaml \ - --set "configurator.executeMigration=${MIGRATION_NAME}" \ + -f /tmp/.current-values.yaml \ + -f ${NEW_CHART_VALUES} \ + --set "configurator.executeMigration=${MIGRATION_NAME}" \ + && echo "Removing migrations..." \ && helm -n ${NS} upgrade ${RELEASE_NAME} ./ \ - -f ${NEW_CHART_VALUES} \ - -f /tmp/.current-values.yaml \ - --set "configurator.executeMigration=" + -f /tmp/.current-values.yaml \ + -f ${NEW_CHART_VALUES} \ + --set "configurator.executeMigration=" ) diff --git a/bin/setup.sh b/bin/setup.sh index ccdcf627b..56431f2e1 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -108,7 +108,6 @@ COMPOSE_PROFILES="${COMPOSE_PROFILES},setup" docker compose run --rm -T --entryp docker compose restart keycloak docker compose run --rm dockerize -wait http://keycloak:8080 -timeout 200s -docker compose run --rm --entrypoint="" configurator /bin/ash -c env docker compose run --rm configurator configure -vvv echo "Done." diff --git a/configurator/src/Command/Migration20230807Command.php b/configurator/src/Command/Migration20230807Command.php index d22263845..e2eeef261 100644 --- a/configurator/src/Command/Migration20230807Command.php +++ b/configurator/src/Command/Migration20230807Command.php @@ -19,8 +19,8 @@ public function __construct( private readonly KeycloakManager $keycloakManager, private readonly array $symfonyApplications, private readonly DoctrineConnectionManager $connections, - ) - { + private readonly string $keycloakRealm, + ) { parent::__construct(); } @@ -29,6 +29,8 @@ public function execute(InputInterface $input, OutputInterface $output): int $apps = $this->symfonyApplications; $apps[] = 'auth'; + $this->migrateIdP(); + foreach ($apps as $app) { $output->writeln(sprintf('Migrating %s', $app)); $connection = $this->connections->getConnection($app); @@ -58,12 +60,12 @@ public function execute(InputInterface $input, OutputInterface $output): int continue; } if ('auth' === $app && in_array($row['id'], [ - 'databox-app', - 'expose-app', - 'uploader-app', - 'databox-admin', - 'expose-admin', - 'uploader-admin', + 'databox-app', + 'expose-app', + 'uploader-app', + 'databox-admin', + 'expose-admin', + 'uploader-admin', ], true)) { continue; } @@ -134,7 +136,7 @@ public function execute(InputInterface $input, OutputInterface $output): int $user = $this->keycloakManager->createUser([ 'createdTimestamp' => (new \DateTimeImmutable($row['created_at']))->getTimestamp(), 'username' => $row['username'], - 'email' => $row['username'], + 'email' => str_contains($row['username'], '@') ? $row['username'] : null, 'emailVerified' => $row['email_verified'], 'enabled' => $row['enabled'], 'attributes' => [ @@ -142,7 +144,7 @@ public function execute(InputInterface $input, OutputInterface $output): int ], 'requiredActions' => [ 'UPDATE_PASSWORD', - ] + ], ]); $userMap[$row['id']] = $user['id']; @@ -159,6 +161,35 @@ public function execute(InputInterface $input, OutputInterface $output): int $this->keycloakManager->addRolesToUser($user['id'], $realmRoles); } + $samlIdentities = $connection->fetchAllAssociative('SELECT +user_id, +provider, +attributes +FROM "saml_identity"'); + foreach ($samlIdentities as $row) { + $attributes = json_decode($row['attributes'], true, 512, JSON_THROW_ON_ERROR); + $username = $this->extractUsernameFromAttributes($attributes); + + $this->keycloakManager->linkAccountToIdentityProvider($userMap[$row['user_id']], $row['provider'], [ + 'userId' => $username, + 'userName' => $username, + ]); + } + + $oauthUsers = $connection->fetchAllAssociative('SELECT +user_id, +identifier, +provider +FROM "external_access_token"'); + foreach ($oauthUsers as $row) { + $username = $row['identifier']; + + $this->keycloakManager->linkAccountToIdentityProvider($userMap[$row['user_id']], $row['provider'], [ + 'userId' => $username, + 'userName' => $username, + ]); + } + $this->replaceInDb([ 'databox' => [ 'asset_data_template' => [ @@ -213,8 +244,8 @@ public function execute(InputInterface $input, OutputInterface $output): int 'databox' => [ 'access_control_entry' => [ 'user_id', - ] - ] + ], + ], ], $groupMap); @@ -237,4 +268,115 @@ private function replaceInDb(array $tableMap, array $valueMap): void } } } + + private function migrateIdP(): void + { + $configSrc = '/configs/config.json'; + if (!file_exists($configSrc)) { + return; + } + + $config = json_decode(file_get_contents($configSrc), true, 512, JSON_THROW_ON_ERROR); + + foreach ($config['auth']['identity_providers'] ?? [] as $idp) { + $alias = $idp['name']; + $options = $idp['options']; + + $normalizeOAuthUrl = function (string $key, string $default) use ($options, $alias): string { + if (isset($options[$key])) { + return str_replace('{base_url}', $options['base_url'] ?? '', $options[$key]); + } + + if (!isset($options['base_url'])) { + throw new \InvalidArgumentException(sprintf('Missing "base_url" for IdP "%s"', $alias)); + } + + return $options['base_url'].$default; + }; + + $idpType = $idp['type']; + if ('oauth' === $idpType) { + $idpType = 'oidc'; + } + + $config = match ($idpType) { + 'saml' => [ + 'allowCreate' => 'true', + 'guiOrder' => '', + 'entityId' => getenv('KEYCLOAK_URL').'/realms/'.$this->keycloakRealm, + 'idpEntityId' => $options['entity_id'], + 'singleSignOnServiceUrl' => $options['sso_url'], + 'singleLogoutServiceUrl' => '', + 'attributeConsumingServiceName' => '', + 'backchannelSupported' => 'false', + 'nameIDPolicyFormat' => 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', + 'principalType' => 'Subject NameID', + 'postBindingResponse' => 'false', + 'postBindingAuthnRequest' => 'false', + 'postBindingLogout' => 'false', + 'wantAuthnRequestsSigned' => 'false', + 'wantAssertionsSigned' => 'false', + 'wantAssertionsEncrypted' => 'false', + 'forceAuthn' => 'false', + 'validateSignature' => 'false', + 'signSpMetadata' => 'false', + 'loginHint' => 'false', + 'allowedClockSkew' => 0, + 'attributeConsumingServiceIndex' => 0, + ], + 'oidc' => [ + 'allowCreate' => true, + 'authorizationUrl' => $normalizeOAuthUrl('authorization_url', '/auth'), + 'tokenUrl' => $normalizeOAuthUrl('token_url', '/token'), + 'userInfoUrl' => $normalizeOAuthUrl('userinfo', '/userinfo'), + 'clientAssertionSigningAlg' => '', + 'clientAuthMethod' => 'client_secret_post', + 'validateSignature' => 'false', + 'clientId' => $options['client_id'], + 'clientSecret' => $options['client_secret'], + ], + }; + + $data = [ + 'alias' => $alias, + 'config' => $config, + 'displayName' => $idp['title'], + 'providerId' => $idpType, +// 'enabled' => true, +// 'trustEmail' => true, + ]; + + $this->keycloakManager->createIdentityProvider($data); + + if (isset($idp['group_jq_normalizer'])) { + $this->keycloakManager->createIdpMapper($alias, [ + 'name' => 'groups', + 'identityProviderAlias' => $alias, + 'identityProviderMapper' => 'jq-groups-idp-mapper', + 'config' => [ + 'syncMode' => 'FORCE', + 'jq_filter' => $idp['group_jq_normalizer'], + ], + ]); + } + } + } + + private function extractUsernameFromAttributes(array $attributes): string + { + foreach ([ + 'username', + 'email', + ] as $key) { + if (!empty($attributes[$key])) { + if (is_string($attributes[$key])) { + return $attributes[$key]; + } elseif (is_array($attributes[$key])) { + return $attributes[$key][0]; + } + } + } + + throw new \InvalidArgumentException(sprintf('Cannot extract username in attributes: %s', print_r($attributes, true))); + } } diff --git a/configurator/src/Configurator/Vendor/Keycloak/KeycloakManager.php b/configurator/src/Configurator/Vendor/Keycloak/KeycloakManager.php index effe62b64..b35ad5f2e 100644 --- a/configurator/src/Configurator/Vendor/Keycloak/KeycloakManager.php +++ b/configurator/src/Configurator/Vendor/Keycloak/KeycloakManager.php @@ -6,6 +6,7 @@ use App\Util\HttpClientUtil; use App\Util\UriTemplate; +use Symfony\Component\HttpClient\Exception\ClientException; use Symfony\Contracts\HttpClient\HttpClientInterface; use Symfony\Contracts\HttpClient\ResponseInterface; @@ -56,12 +57,13 @@ private function getToken(): string public function createRealm(): void { - HttpClientUtil::catchHttpCode(fn() => $this->getAuthenticatedClient()->request('POST', '', [ - 'json' => [ - 'realm' => $this->keycloakRealm, - 'enabled' => true, - ], - ])->getContent(), 409); + $data = [ + 'realm' => $this->keycloakRealm, + 'enabled' => true, + ]; + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient()->request('POST', '', [ + 'json' => $data, + ])->getContent(), 409, $data); } protected function getClients(string $realm = null): array @@ -114,11 +116,11 @@ public function createScope(string $name, array $data = []): void ]); } - HttpClientUtil::catchHttpCode(fn() => $this->getAuthenticatedClient() + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() ->request('PUT', UriTemplate::resolve('{realm}/default-default-client-scopes/{id}', [ 'realm' => $this->keycloakRealm, 'id' => $scope['id'], - ])), 409); + ])), 409, []); } @@ -148,12 +150,12 @@ public function addScopeToClient(string $scope, string $clientId): void { $scopeData = $this->getScopeByName($scope); - HttpClientUtil::catchHttpCode(fn () => $this->getAuthenticatedClient() + HttpClientUtil::debugError(fn () => $this->getAuthenticatedClient() ->request('PUT', UriTemplate::resolve('{realm}/clients/{clientId}/default-client-scopes/{scopeId}', [ 'realm' => $this->keycloakRealm, 'clientId' => $clientId, 'scopeId' => $scopeData['id'], - ])), 409); + ])), 409, []); } public function addServiceAccountRole( @@ -270,12 +272,12 @@ public function configureClientClaim( public function createUser(array $data): array { - HttpClientUtil::catchHttpCode(fn() => $this->getAuthenticatedClient() + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() ->request('POST', UriTemplate::resolve('{realm}/users', [ 'realm' => $this->keycloakRealm, ]), [ 'json' => $data, - ]), 409); + ]), 409, $data); $user = $this->getUsers([ 'query' => [ @@ -348,16 +350,18 @@ public function getGroups(array $options = []): array public function createRole(string $name, string $description): void { - HttpClientUtil::catchHttpCode(fn() => $this->getAuthenticatedClient() + $data = [ + 'name' => $name, + 'clientRole' => false, + 'description' => $description, + ]; + + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() ->request('POST', UriTemplate::resolve('{realm}/roles', [ 'realm' => $this->keycloakRealm, ]), [ - 'json' => [ - 'name' => $name, - 'clientRole' => false, - 'description' => $description, - ], - ]), 409); + 'json' => $data, + ]), 409, $data); } public function getRealmRoles(): array @@ -412,13 +416,13 @@ public function addRolesToUser(string $userId, array $roleNames): void }, $roleNames); $roles = array_filter($roles, fn (array|null $r): bool => null !== $r); - HttpClientUtil::catchHttpCode(fn() => $this->getAuthenticatedClient() + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() ->request('POST', UriTemplate::resolve('{realm}/users/{userId}/role-mappings/realm', [ 'realm' => $this->keycloakRealm, 'userId' => $userId, ]), [ 'json' => $roles, - ]), 409); + ]), 409, $roles); } public function addClientRolesToUser(string $userId, array $roleNames): void @@ -442,24 +446,24 @@ public function addClientRolesToUser(string $userId, array $roleNames): void }, $roleNames); $roles = array_filter($roles, fn (array|null $r): bool => null !== $r); - HttpClientUtil::catchHttpCode(fn() => $this->getAuthenticatedClient() + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() ->request('POST', UriTemplate::resolve('{realm}/users/{userId}/role-mappings/clients/{realmClientId}', [ 'realm' => $this->keycloakRealm, 'userId' => $userId, 'realmClientId' => $realmClient['id'], ]), [ 'json' => $roles, - ]), 409); + ]), 409, $roles); } public function addUserToGroup(string $userId, string $groupId): void { - HttpClientUtil::catchHttpCode(fn() => $this->getAuthenticatedClient() + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() ->request('PUT', UriTemplate::resolve('{realm}/users/{userId}/groups/{groupId}', [ 'realm' => $this->keycloakRealm, 'userId' => $userId, 'groupId' => $groupId, - ])), 409); + ])), 409, []); } public function putRealm(array $data): ResponseInterface @@ -471,4 +475,74 @@ public function putRealm(array $data): ResponseInterface 'json' => $data, ]); } + + public function createIdentityProvider(array $data): array + { + $idpAlias = $data['alias']; + $existingIdp = $this->getIdentityProviderByAlias($idpAlias); + + if ($existingIdp) { + $this->getAuthenticatedClient() + ->request('PUT', UriTemplate::resolve('{realm}/identity-provider/instances/{alias}', [ + 'realm' => $this->keycloakRealm, + 'alias' => $existingIdp['alias'], + ]), [ + 'json' => $data, + ]); + } else { + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() + ->request('POST', UriTemplate::resolve('{realm}/identity-provider/instances', [ + 'realm' => $this->keycloakRealm, + ]), [ + 'json' => $data, + ]), null, $data); + } + + $idp = $this->getIdentityProviderByAlias($idpAlias); + if ($idp['alias'] !== $idpAlias) { + throw new \InvalidArgumentException(sprintf('Invalid IdP "%s" "%s"', $idp['alias'], $idpAlias)); + } + + return $idp; + } + + public function getIdentityProviderByAlias(string $alias): ?array + { + try { + return $this->getAuthenticatedClient() + ->request('GET', UriTemplate::resolve('{realm}/identity-provider/instances/{alias}', [ + 'realm' => $this->keycloakRealm, + 'alias' => $alias, + ]))->toArray(); + } catch (ClientException $e) { + if ($e->getResponse()->getStatusCode() !== 404) { + throw $e; + } + } + + return null; + } + + public function createIdpMapper(string $idpAlias, array $data): void + { + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() + ->request('POST', UriTemplate::resolve('{realm}/identity-provider/instances/{alias}/mappers', [ + 'realm' => $this->keycloakRealm, + 'alias' => $idpAlias, + ]), [ + 'json' => $data, + ]), 400, $data); + } + + public function linkAccountToIdentityProvider(string $userId, string $idpAlias, array $data): void + { + HttpClientUtil::debugError(fn() => $this->getAuthenticatedClient() + ->request('POST', UriTemplate::resolve('{realm}/users/{userId}/federated-identity/{alias}', [ + 'realm' => $this->keycloakRealm, + 'userId' => $userId, + 'alias' => $idpAlias, + ]), [ + 'json' => $data, + ]), 409, $data); + } } diff --git a/configurator/src/Util/HttpClientUtil.php b/configurator/src/Util/HttpClientUtil.php index b79847021..395f2c69e 100644 --- a/configurator/src/Util/HttpClientUtil.php +++ b/configurator/src/Util/HttpClientUtil.php @@ -8,14 +8,23 @@ abstract class HttpClientUtil { - public static function catchHttpCode(callable $handler, int $httpCode) + public static function debugError(callable $handler, ?int $ignoreHttpCode = null, ?array $data = null): void { try { $handler(); } catch (ClientException $e) { - if ($e->getResponse()->getStatusCode() !== $httpCode) { - throw $e; + if (null !== $ignoreHttpCode && $ignoreHttpCode === $e->getResponse()->getStatusCode()) { + return; } + + $error = $e->getResponse()->getContent(false); + + throw new \InvalidArgumentException(sprintf( + '%s: %s%s', + $e->getMessage(), + $error, + null !== $data ? ' (with data: '.print_r($data, true).')' : '', + ), 0, $e); } } } diff --git a/docker-compose.saml.yml b/docker-compose.saml.yml index de90139d7..82c63cb32 100644 --- a/docker-compose.saml.yml +++ b/docker-compose.saml.yml @@ -7,9 +7,9 @@ services: - internal environment: - SIMPLESAMLPHP_URL=${SAML_URL} - - SIMPLESAMLPHP_SP_ENTITY_ID=https://api-auth.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/saml/metadata/idp-test - - SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE=https://api-auth.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/saml/acs - - SIMPLESAMLPHP_SP_SINGLE_LOGOUT_SERVICE=https://api-auth.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/saml/logout + - SIMPLESAMLPHP_SP_ENTITY_ID=https://keycloak.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/realms/phrasea + - SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE=https://keycloak.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/realms/phrasea/broker/idp-test/endpoint + - SIMPLESAMLPHP_SP_SINGLE_LOGOUT_SERVICE=https://keycloak.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/realms/phrasea/broker/idp-test/endpoint labels: - "traefik.enable=true" - "traefik.http.routers.saml-idp.rule=Host(`saml-idp.${PHRASEA_DOMAIN}`)" @@ -22,9 +22,9 @@ services: - internal environment: - SIMPLESAMLPHP_URL=${SAML2_URL} - - SIMPLESAMLPHP_SP_ENTITY_ID=https://api-auth.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/saml/metadata/idp-test2 - - SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE=https://api-auth.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/saml/acs - - SIMPLESAMLPHP_SP_SINGLE_LOGOUT_SERVICE=https://api-auth.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/saml/logout + - SIMPLESAMLPHP_SP_ENTITY_ID=https://keycloak.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/realms/phrasea + - SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE=https://keycloak.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/realms/phrasea/broker/idp-test/endpoint + - SIMPLESAMLPHP_SP_SINGLE_LOGOUT_SERVICE=https://keycloak.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}/realms/phrasea/broker/idp-test/endpoint labels: - "traefik.enable=true" - "traefik.http.routers.saml-idp2.rule=Host(`saml-idp2.${PHRASEA_DOMAIN}`)" diff --git a/docker-compose.yml b/docker-compose.yml index 6ae35facf..cfe7a3ffa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1013,6 +1013,8 @@ services: - DEFAULT_ADMIN_PASSWORD extra_hosts: - keycloak.${PHRASEA_DOMAIN}:${PS_GATEWAY_IP} + volumes: + - ./configs:/configs volumes: db_vol: