diff --git a/components/renku_data_services/app_config/config.py b/components/renku_data_services/app_config/config.py index fa2873eff..85ce52e2e 100644 --- a/components/renku_data_services/app_config/config.py +++ b/components/renku_data_services/app_config/config.py @@ -52,7 +52,7 @@ from renku_data_services.message_queue.interface import IMessageQueue from renku_data_services.message_queue.redis_queue import RedisQueue from renku_data_services.namespace.db import GroupRepository -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.config import NotebooksConfig from renku_data_services.platform.db import PlatformRepository from renku_data_services.project.db import ProjectMemberRepository, ProjectRepository from renku_data_services.repositories.db import GitRepositoriesRepository @@ -145,7 +145,7 @@ class Config: kc_api: IKeycloakAPI message_queue: IMessageQueue gitlab_url: str | None - nb_config: _NotebooksConfig + nb_config: NotebooksConfig secrets_service_public_key: rsa.RSAPublicKey """The public key of the secrets service, used to encrypt user secrets that only it can decrypt.""" @@ -497,7 +497,7 @@ def from_env(cls, prefix: str = "") -> "Config": sentry = SentryConfig.from_env(prefix) trusted_proxies = TrustedProxiesConfig.from_env(prefix) message_queue = RedisQueue(redis) - nb_config = _NotebooksConfig.from_env(db) + nb_config = NotebooksConfig.from_env(db) return cls( version=version, diff --git a/components/renku_data_services/base_models/core.py b/components/renku_data_services/base_models/core.py index 1f6a76cbb..fe4f32fe7 100644 --- a/components/renku_data_services/base_models/core.py +++ b/components/renku_data_services/base_models/core.py @@ -31,6 +31,11 @@ def is_authenticated(self) -> bool: """Indicates whether the user has successfully logged in.""" return self.id is not None + @property + def is_anonymous(self) -> bool: + """Indicates whether the user is anonymous.""" + return isinstance(self, AnonymousAPIUser) + def get_full_name(self) -> str | None: """Generate the closest thing to a full name if the full name field is not set.""" full_name = self.full_name or " ".join(filter(None, (self.first_name, self.last_name))) diff --git a/components/renku_data_services/crc/db.py b/components/renku_data_services/crc/db.py index a8e95d795..4b42b3edd 100644 --- a/components/renku_data_services/crc/db.py +++ b/components/renku_data_services/crc/db.py @@ -293,6 +293,13 @@ async def get_classes( orms = res.scalars().all() return [orm.dump() for orm in orms] + async def get_resource_class(self, api_user: base_models.APIUser, id: int) -> models.ResourceClass: + """Get a specific resource class by its ID.""" + classes = await self.get_classes(api_user, id) + if len(classes) == 0: + raise errors.MissingResourceError(message=f"The resource class with ID {id} cannot be found", quiet=True) + return classes[0] + @_only_admins async def insert_resource_class( self, diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/git_proxy.py b/components/renku_data_services/notebooks/api/amalthea_patches/git_proxy.py index 3773c57ea..38f738321 100644 --- a/components/renku_data_services/notebooks/api/amalthea_patches/git_proxy.py +++ b/components/renku_data_services/notebooks/api/amalthea_patches/git_proxy.py @@ -6,48 +6,53 @@ from kubernetes import client +from renku_data_services.base_models.core import AnonymousAPIUser, AuthenticatedAPIUser from renku_data_services.notebooks.api.amalthea_patches.utils import get_certificates_volume_mounts +from renku_data_services.notebooks.api.classes.repository import GitProvider, Repository +from renku_data_services.notebooks.config import NotebooksConfig if TYPE_CHECKING: # NOTE: If these are directly imported then you get circular imports. from renku_data_services.notebooks.api.classes.server import UserServer -async def main_container(server: "UserServer") -> client.V1Container | None: +async def main_container( + user: AnonymousAPIUser | AuthenticatedAPIUser, + config: NotebooksConfig, + repositories: list[Repository], + git_providers: list[GitProvider], +) -> client.V1Container | None: """The patch that adds the git proxy container to a session statefulset.""" - repositories = await server.repositories() - if not server.user.is_authenticated or not repositories: + if not user.is_authenticated or not repositories or user.access_token is None or user.refresh_token is None: return None etc_cert_volume_mount = get_certificates_volume_mounts( - server.config, + config, custom_certs=False, etc_certs=True, read_only_etc_certs=True, ) prefix = "GIT_PROXY_" - git_providers = await server.git_providers() - repositories = await server.repositories() env = [ - client.V1EnvVar(name=f"{prefix}PORT", value=str(server.config.sessions.git_proxy.port)), - client.V1EnvVar(name=f"{prefix}HEALTH_PORT", value=str(server.config.sessions.git_proxy.health_port)), + client.V1EnvVar(name=f"{prefix}PORT", value=str(config.sessions.git_proxy.port)), + client.V1EnvVar(name=f"{prefix}HEALTH_PORT", value=str(config.sessions.git_proxy.health_port)), client.V1EnvVar( name=f"{prefix}ANONYMOUS_SESSION", - value="false" if server.user.is_authenticated else "true", + value="false" if user.is_authenticated else "true", ), - client.V1EnvVar(name=f"{prefix}RENKU_ACCESS_TOKEN", value=str(server.user.access_token)), - client.V1EnvVar(name=f"{prefix}RENKU_REFRESH_TOKEN", value=str(server.user.refresh_token)), - client.V1EnvVar(name=f"{prefix}RENKU_REALM", value=server.config.keycloak_realm), + client.V1EnvVar(name=f"{prefix}RENKU_ACCESS_TOKEN", value=str(user.access_token)), + client.V1EnvVar(name=f"{prefix}RENKU_REFRESH_TOKEN", value=str(user.refresh_token)), + client.V1EnvVar(name=f"{prefix}RENKU_REALM", value=config.keycloak_realm), client.V1EnvVar( name=f"{prefix}RENKU_CLIENT_ID", - value=str(server.config.sessions.git_proxy.renku_client_id), + value=str(config.sessions.git_proxy.renku_client_id), ), client.V1EnvVar( name=f"{prefix}RENKU_CLIENT_SECRET", - value=str(server.config.sessions.git_proxy.renku_client_secret), + value=str(config.sessions.git_proxy.renku_client_secret), ), - client.V1EnvVar(name=f"{prefix}RENKU_URL", value="https://" + server.config.sessions.ingress.host), + client.V1EnvVar(name=f"{prefix}RENKU_URL", value="https://" + config.sessions.ingress.host), client.V1EnvVar( name=f"{prefix}REPOSITORIES", value=json.dumps([asdict(repo) for repo in repositories]), @@ -60,7 +65,7 @@ async def main_container(server: "UserServer") -> client.V1Container | None: ), ] container = client.V1Container( - image=server.config.sessions.git_proxy.image, + image=config.sessions.git_proxy.image, security_context={ "fsGroup": 100, "runAsGroup": 1000, @@ -73,14 +78,14 @@ async def main_container(server: "UserServer") -> client.V1Container | None: liveness_probe={ "httpGet": { "path": "/health", - "port": server.config.sessions.git_proxy.health_port, + "port": config.sessions.git_proxy.health_port, }, "initialDelaySeconds": 3, }, readiness_probe={ "httpGet": { "path": "/health", - "port": server.config.sessions.git_proxy.health_port, + "port": config.sessions.git_proxy.health_port, }, "initialDelaySeconds": 3, }, @@ -98,7 +103,8 @@ async def main(server: "UserServer") -> list[dict[str, Any]]: if not server.user.is_authenticated or not repositories: return [] - container = await main_container(server) + git_providers = await server.git_providers() + container = await main_container(server.user, server.config, repositories, git_providers) if not container: return [] diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py b/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py index fbb789372..638dd6171 100644 --- a/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py +++ b/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py @@ -3,72 +3,77 @@ import json import os from dataclasses import asdict -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any from kubernetes import client +from renku_data_services.base_models.core import AnonymousAPIUser, AuthenticatedAPIUser from renku_data_services.notebooks.api.amalthea_patches.utils import get_certificates_volume_mounts -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.api.classes.repository import GitProvider, Repository +from renku_data_services.notebooks.config import NotebooksConfig if TYPE_CHECKING: # NOTE: If these are directly imported then you get circular imports. from renku_data_services.notebooks.api.classes.server import UserServer -async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None: +async def git_clone_container_v2( + user: AuthenticatedAPIUser | AnonymousAPIUser, + config: NotebooksConfig, + repositories: list[Repository], + git_providers: list[GitProvider], + workspace_mount_path: PurePosixPath, + work_dir: PurePosixPath, + lfs_auto_fetch: bool = False, +) -> dict[str, Any] | None: """Returns the specification for the container that clones the user's repositories for new operator.""" amalthea_session_work_volume: str = "amalthea-volume" - repositories = await server.repositories() if not repositories: return None etc_cert_volume_mount = get_certificates_volume_mounts( - server.config, + config, custom_certs=False, etc_certs=True, read_only_etc_certs=True, ) - user_is_anonymous = not server.user.is_authenticated prefix = "GIT_CLONE_" env = [ - { - "name": f"{prefix}WORKSPACE_MOUNT_PATH", - "value": server.workspace_mount_path.as_posix(), - }, + {"name": f"{prefix}WORKSPACE_MOUNT_PATH", "value": workspace_mount_path.as_posix()}, { "name": f"{prefix}MOUNT_PATH", - "value": server.work_dir.as_posix(), + "value": work_dir.as_posix(), }, { "name": f"{prefix}LFS_AUTO_FETCH", - "value": "1" if server.server_options.lfs_auto_fetch else "0", + "value": "1" if lfs_auto_fetch else "0", }, { "name": f"{prefix}USER__USERNAME", - "value": server.user.email, + "value": user.email, }, { "name": f"{prefix}USER__RENKU_TOKEN", - "value": str(server.user.access_token), + "value": str(user.access_token), }, - {"name": f"{prefix}IS_GIT_PROXY_ENABLED", "value": "0" if user_is_anonymous else "1"}, + {"name": f"{prefix}IS_GIT_PROXY_ENABLED", "value": "0" if user.is_anonymous else "1"}, { "name": f"{prefix}SENTRY__ENABLED", - "value": str(server.config.sessions.git_clone.sentry.enabled).lower(), + "value": str(config.sessions.git_clone.sentry.enabled).lower(), }, { "name": f"{prefix}SENTRY__DSN", - "value": server.config.sessions.git_clone.sentry.dsn, + "value": config.sessions.git_clone.sentry.dsn, }, { "name": f"{prefix}SENTRY__ENVIRONMENT", - "value": server.config.sessions.git_clone.sentry.env, + "value": config.sessions.git_clone.sentry.env, }, { "name": f"{prefix}SENTRY__SAMPLE_RATE", - "value": str(server.config.sessions.git_clone.sentry.sample_rate), + "value": str(config.sessions.git_clone.sentry.sample_rate), }, {"name": "SENTRY_RELEASE", "value": os.environ.get("SENTRY_RELEASE")}, { @@ -80,12 +85,12 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None: "value": str(Path(etc_cert_volume_mount[0]["mountPath"]) / "ca-certificates.crt"), }, ] - if server.user.is_authenticated: - if server.user.email: + if user.is_authenticated: + if user.email: env.append( - {"name": f"{prefix}USER__EMAIL", "value": server.user.email}, + {"name": f"{prefix}USER__EMAIL", "value": user.email}, ) - full_name = server.user.get_full_name() + full_name = user.get_full_name() if full_name: env.append( { @@ -105,7 +110,8 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None: ) # Set up git providers - required_git_providers = await server.required_git_providers() + required_provider_ids: set[str] = {r.provider for r in repositories if r.provider} + required_git_providers = [p for p in git_providers if p.id in required_provider_ids] for idx, provider in enumerate(required_git_providers): obj_env = f"{prefix}GIT_PROVIDERS_{idx}_" data = dict(id=provider.id, access_token_url=provider.access_token_url) @@ -117,7 +123,7 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None: ) return { - "image": server.config.sessions.git_clone.image, + "image": config.sessions.git_clone.image, "name": "git-clone", "resources": { "requests": { @@ -134,7 +140,7 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None: }, "volumeMounts": [ { - "mountPath": server.workspace_mount_path.as_posix(), + "mountPath": workspace_mount_path.as_posix(), "name": amalthea_session_work_volume, }, *etc_cert_volume_mount, @@ -156,7 +162,6 @@ async def git_clone_container(server: "UserServer") -> dict[str, Any] | None: read_only_etc_certs=True, ) - user_is_anonymous = not server.user.is_authenticated prefix = "GIT_CLONE_" env = [ { @@ -179,7 +184,7 @@ async def git_clone_container(server: "UserServer") -> dict[str, Any] | None: "name": f"{prefix}USER__RENKU_TOKEN", "value": str(server.user.access_token), }, - {"name": f"{prefix}IS_GIT_PROXY_ENABLED", "value": "0" if user_is_anonymous else "1"}, + {"name": f"{prefix}IS_GIT_PROXY_ENABLED", "value": "0" if server.user.is_anonymous else "1"}, { "name": f"{prefix}SENTRY__ENABLED", "value": str(server.config.sessions.git_clone.sentry.enabled).lower(), @@ -288,7 +293,7 @@ async def git_clone(server: "UserServer") -> list[dict[str, Any]]: ] -def certificates_container(config: _NotebooksConfig) -> tuple[client.V1Container, list[client.V1Volume]]: +def certificates_container(config: NotebooksConfig) -> tuple[client.V1Container, list[client.V1Volume]]: """The specification for the container that setups self signed CAs.""" init_container = client.V1Container( name="init-certificates", @@ -321,7 +326,7 @@ def certificates_container(config: _NotebooksConfig) -> tuple[client.V1Container return (init_container, [volume_etc_certs, volume_custom_certs]) -def certificates(config: _NotebooksConfig) -> list[dict[str, Any]]: +def certificates(config: NotebooksConfig) -> list[dict[str, Any]]: """Add a container that initializes custom certificate authorities for a session.""" container, vols = certificates_container(config) api_client = client.ApiClient() diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/ssh.py b/components/renku_data_services/notebooks/api/amalthea_patches/ssh.py index da565966d..9bb3d99e3 100644 --- a/components/renku_data_services/notebooks/api/amalthea_patches/ssh.py +++ b/components/renku_data_services/notebooks/api/amalthea_patches/ssh.py @@ -2,10 +2,10 @@ from typing import Any -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.config import NotebooksConfig -def main(config: _NotebooksConfig) -> list[dict[str, Any]]: +def main(config: NotebooksConfig) -> list[dict[str, Any]]: """Adds the required configuration to the session statefulset for SSH access.""" if not config.sessions.ssh.enabled: return [] diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/utils.py b/components/renku_data_services/notebooks/api/amalthea_patches/utils.py index fccd7ac58..650970977 100644 --- a/components/renku_data_services/notebooks/api/amalthea_patches/utils.py +++ b/components/renku_data_services/notebooks/api/amalthea_patches/utils.py @@ -4,11 +4,11 @@ from kubernetes import client -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.config import NotebooksConfig def get_certificates_volume_mounts( - config: _NotebooksConfig, + config: NotebooksConfig, etc_certs: bool = True, custom_certs: bool = True, read_only_etc_certs: bool = False, diff --git a/components/renku_data_services/notebooks/api/classes/server.py b/components/renku_data_services/notebooks/api/classes/server.py index 147b02e7c..3add6cec6 100644 --- a/components/renku_data_services/notebooks/api/classes/server.py +++ b/components/renku_data_services/notebooks/api/classes/server.py @@ -25,7 +25,7 @@ from renku_data_services.notebooks.api.classes.repository import GitProvider, Repository from renku_data_services.notebooks.api.schemas.secrets import K8sUserSecrets from renku_data_services.notebooks.api.schemas.server_options import ServerOptions -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.config import NotebooksConfig from renku_data_services.notebooks.crs import JupyterServerV1Alpha1 from renku_data_services.notebooks.errors.programming import DuplicateEnvironmentVariableError from renku_data_services.notebooks.errors.user import MissingResourceError @@ -46,7 +46,7 @@ def __init__( k8s_client: K8sClient, workspace_mount_path: PurePosixPath, work_dir: PurePosixPath, - config: _NotebooksConfig, + config: NotebooksConfig, internal_gitlab_user: APIUser, using_default_image: bool = False, is_image_private: bool = False, @@ -380,7 +380,7 @@ def __init__( k8s_client: K8sClient, workspace_mount_path: PurePosixPath, work_dir: PurePosixPath, - config: _NotebooksConfig, + config: NotebooksConfig, gitlab_project: Project | None, internal_gitlab_user: APIUser, using_default_image: bool = False, @@ -506,7 +506,7 @@ def __init__( workspace_mount_path: PurePosixPath, work_dir: PurePosixPath, repositories: list[Repository], - config: _NotebooksConfig, + config: NotebooksConfig, internal_gitlab_user: APIUser, using_default_image: bool = False, is_image_private: bool = False, diff --git a/components/renku_data_services/notebooks/api/schemas/cloud_storage.py b/components/renku_data_services/notebooks/api/schemas/cloud_storage.py index 5b848f8fa..11ee5a5ca 100644 --- a/components/renku_data_services/notebooks/api/schemas/cloud_storage.py +++ b/components/renku_data_services/notebooks/api/schemas/cloud_storage.py @@ -10,7 +10,7 @@ from renku_data_services.base_models import APIUser from renku_data_services.notebooks.api.classes.cloud_storage import ICloudStorageRequest -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.config import NotebooksConfig _sanitize_for_serialization = client.ApiClient().sanitize_for_serialization @@ -48,7 +48,7 @@ def __init__( readonly: bool, mount_folder: str, name: Optional[str], - config: _NotebooksConfig, + config: NotebooksConfig, ) -> None: """Creates a cloud storage instance without validating the configuration.""" self.config = config @@ -66,7 +66,7 @@ async def storage_from_schema( internal_gitlab_user: APIUser, project_id: int, work_dir: PurePosixPath, - config: _NotebooksConfig, + config: NotebooksConfig, ) -> Self: """Create storage object from request.""" name = None diff --git a/components/renku_data_services/notebooks/api/schemas/servers_get.py b/components/renku_data_services/notebooks/api/schemas/servers_get.py index 77c7b09aa..110356c92 100644 --- a/components/renku_data_services/notebooks/api/schemas/servers_get.py +++ b/components/renku_data_services/notebooks/api/schemas/servers_get.py @@ -11,7 +11,7 @@ from renku_data_services.notebooks.api.classes.server_manifest import UserServerManifest from renku_data_services.notebooks.api.schemas.cloud_storage import LaunchNotebookResponseCloudStorage from renku_data_services.notebooks.api.schemas.custom_fields import ByteSizeField, CpuField, GpuField, LowercaseString -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.config import NotebooksConfig from renku_data_services.notebooks.config.static import _ServersGetEndpointAnnotations @@ -134,7 +134,7 @@ class Meta: image = fields.Str() @staticmethod - def format_user_pod_data(server: UserServerManifest, config: _NotebooksConfig) -> dict[str, Any]: + def format_user_pod_data(server: UserServerManifest, config: NotebooksConfig) -> dict[str, Any]: """Convert and format a server manifest object into what the API requires.""" def get_failed_container_exit_code(container_status: dict[str, Any]) -> int | str: diff --git a/components/renku_data_services/notebooks/blueprints.py b/components/renku_data_services/notebooks/blueprints.py index 46d965fa4..c0ef15abb 100644 --- a/components/renku_data_services/notebooks/blueprints.py +++ b/components/renku_data_services/notebooks/blueprints.py @@ -48,7 +48,7 @@ ServersGetResponse, ) from renku_data_services.notebooks.api.schemas.servers_patch import PatchServerStatusEnum -from renku_data_services.notebooks.config import _NotebooksConfig +from renku_data_services.notebooks.config import NotebooksConfig from renku_data_services.notebooks.crs import ( AmaltheaSessionSpec, AmaltheaSessionV1Alpha1, @@ -93,7 +93,7 @@ class NotebooksBP(CustomBlueprint): """Handlers for manipulating notebooks.""" authenticator: Authenticator - nb_config: _NotebooksConfig + nb_config: NotebooksConfig git_repo: GitRepositoriesRepository internal_gitlab_authenticator: base_models.Authenticator rp_repo: ResourcePoolRepository @@ -273,7 +273,7 @@ async def _launch_notebook_old( @staticmethod async def launch_notebook_helper( - nb_config: _NotebooksConfig, + nb_config: NotebooksConfig, server_name: str, server_class: type[UserServer], user: AnonymousAPIUser | AuthenticatedAPIUser, @@ -781,7 +781,7 @@ class NotebooksNewBP(CustomBlueprint): authenticator: base_models.Authenticator internal_gitlab_authenticator: base_models.Authenticator - nb_config: _NotebooksConfig + nb_config: NotebooksConfig project_repo: ProjectRepository session_repo: SessionRepository rp_repo: ResourcePoolRepository @@ -811,13 +811,11 @@ async def _handler( image = environment.container_image default_resource_class = await self.rp_repo.get_default_resource_class() if default_resource_class.id is None: - raise errors.ProgrammingError(message="The default reosurce class has to have an ID", quiet=True) + raise errors.ProgrammingError(message="The default resource class has to have an ID", quiet=True) resource_class_id = body.resource_class_id or default_resource_class.id - parsed_server_options = await self.nb_config.crc_validator.validate_class_storage( - user, resource_class_id, body.disk_storage - ) + await self.nb_config.crc_validator.validate_class_storage(user, resource_class_id, body.disk_storage) work_dir = environment.working_directory - user_secrets: K8sUserSecrets | None = None + # user_secrets: K8sUserSecrets | None = None # if body.user_secrets: # user_secrets = K8sUserSecrets( # name=server_name, @@ -860,33 +858,13 @@ async def _handler( quiet=True, ) cloud_storage[csr_id] = csr - # repositories = [Repository(i.url, branch=i.branch, commit_sha=i.commit_sha) for i in body.repositories] repositories = [Repository(url=i) for i in project.repositories] secrets_to_create: list[V1Secret] = [] - server = Renku2UserServer( - user=user, - image=image, - project_id=str(launcher.project_id), - launcher_id=body.launcher_id, - server_name=server_name, - server_options=parsed_server_options, - environment_variables={}, - user_secrets=user_secrets, - cloudstorage=[i for i in cloud_storage.values()], - k8s_client=self.nb_config.k8s_v2_client, - workspace_mount_path=work_dir, - work_dir=work_dir, - repositories=repositories, - config=self.nb_config, - using_default_image=self.nb_config.sessions.default_image == image, - is_image_private=False, - internal_gitlab_user=internal_gitlab_user, - ) - # Generate the cloud storage secrets + # Generate the cloud starge secrets data_sources: list[DataSource] = [] for ics, cs in enumerate(cloud_storage.values()): secret_name = f"{server_name}-ds-{ics}" - secrets_to_create.append(cs.secret(secret_name, server.k8s_client.preferred_namespace)) + secrets_to_create.append(cs.secret(secret_name, self.nb_config.k8s_client.preferred_namespace)) data_sources.append( DataSource(mountPath=cs.mount_folder, secretRef=SecretRefWhole(name=secret_name, adopt=True)) ) @@ -905,17 +883,28 @@ async def _handler( ), ) ) - git_clone = await init_containers.git_clone_container_v2(server) + git_providers = await self.nb_config.git_provider_helper.get_providers(user=user) + git_clone = await init_containers.git_clone_container_v2( + user=user, + config=self.nb_config, + repositories=repositories, + git_providers=git_providers, + workspace_mount_path=launcher.environment.mount_directory, + work_dir=launcher.environment.working_directory, + ) if git_clone is not None: session_init_containers.append(InitContainer.model_validate(git_clone)) extra_containers: list[ExtraContainer] = [] - git_proxy_container = await git_proxy.main_container(server) + git_proxy_container = await git_proxy.main_container( + user=user, config=self.nb_config, repositories=repositories, git_providers=git_providers + ) if git_proxy_container is not None: extra_containers.append( ExtraContainer.model_validate(self.nb_config.k8s_v2_client.sanitize(git_proxy_container)) ) - parsed_server_url = urlparse(server.server_url) + base_server_url = self.nb_config.sessions.ingress.base_url(server_name) + base_server_path = self.nb_config.sessions.ingress.base_path(server_name) annotations: dict[str, str] = { "renku.io/project_id": str(launcher.project_id), "renku.io/launcher_id": body.launcher_id, @@ -928,7 +917,7 @@ async def _handler( hibernated=False, session=Session( image=image, - urlPath=parsed_server_url.path, + urlPath=base_server_path, port=environment.port, storage=Storage( className=self.nb_config.sessions.storage.pvs_storage_class, @@ -944,8 +933,8 @@ async def _handler( args=environment.args, shmSize="1G", env=[ - SessionEnvItem(name="RENKU_BASE_URL_PATH", value=parsed_server_url.path), - SessionEnvItem(name="RENKU_BASE_URL", value=server.server_url), + SessionEnvItem(name="RENKU_BASE_URL_PATH", value=base_server_path), + SessionEnvItem(name="RENKU_BASE_URL", value=base_server_url), ], ), ingress=Ingress( @@ -981,7 +970,7 @@ async def _handler( dataSources=data_sources, ), ) - parsed_proxy_url = urlparse(urljoin(server.server_url + "/", "oauth2")) + parsed_proxy_url = urlparse(urljoin(base_server_url + "/", "oauth2")) secret_data = {} if isinstance(user, AuthenticatedAPIUser): secret_data["auth"] = dumps( @@ -991,8 +980,8 @@ async def _handler( "oidc_issuer_url": self.nb_config.sessions.oidc.issuer_url, "session_cookie_minimal": True, "skip_provider_button": True, - "redirect_url": urljoin(server.server_url + "/", "oauth2/callback"), - "cookie_path": parsed_server_url.path, + "redirect_url": urljoin(base_server_url + "/", "oauth2/callback"), + "cookie_path": base_server_path, "proxy_prefix": parsed_proxy_url.path, "authenticated_emails_file": "/authorized_emails/authorized_emails", "client_secret": self.nb_config.sessions.oidc.client_secret, diff --git a/components/renku_data_services/notebooks/config/__init__.py b/components/renku_data_services/notebooks/config/__init__.py index 7b3413690..bfb363bf9 100644 --- a/components/renku_data_services/notebooks/config/__init__.py +++ b/components/renku_data_services/notebooks/config/__init__.py @@ -94,7 +94,9 @@ async def get_providers(self, user: APIUser) -> list[GitProvider]: @dataclass -class _NotebooksConfig: +class NotebooksConfig: + """The notebooks configuration.""" + server_options: _ServerOptionsConfig sessions: _SessionConfig amalthea: _AmaltheaConfig @@ -122,6 +124,7 @@ class _NotebooksConfig: @classmethod def from_env(cls, db_config: DBConfig) -> Self: + """Create a configuration object from environment variables.""" dummy_stores = _parse_str_as_bool(os.environ.get("DUMMY_STORES", False)) sessions_config: _SessionConfig git_config: _GitConfig diff --git a/components/renku_data_services/notebooks/config/dynamic.py b/components/renku_data_services/notebooks/config/dynamic.py index b30388182..0e02ab29e 100644 --- a/components/renku_data_services/notebooks/config/dynamic.py +++ b/components/renku_data_services/notebooks/config/dynamic.py @@ -6,6 +6,7 @@ from enum import Enum from io import StringIO from typing import Any, ClassVar, Optional, Self, Union +from urllib.parse import urlunparse import yaml @@ -257,6 +258,13 @@ def from_env(cls) -> Self: annotations=yaml.safe_load(StringIO(os.environ.get("NB_SESSIONS__INGRESS__ANNOTATIONS", "{}"))), ) + def base_path(self, server_name: str) -> str: + return f"/sessions/{server_name}" + + def base_url(self, server_name: str) -> str: + scheme = "https" if self.tls_secret else "http" + return urlunparse((scheme, self.host, self.base_path(server_name), None, None, None)) + @dataclass class _GenericCullingConfig: diff --git a/components/renku_data_services/notebooks/crs.py b/components/renku_data_services/notebooks/crs.py index 76ea08dc3..1a548ff75 100644 --- a/components/renku_data_services/notebooks/crs.py +++ b/components/renku_data_services/notebooks/crs.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Any, cast -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse, urlunparse from kubernetes.utils import parse_quantity from pydantic import BaseModel, Field, field_validator @@ -170,12 +170,8 @@ def as_apispec(self) -> apispec.SessionResponse: "because it is missing the spec.session.resources field" ) url = "None" - if self.status.url is None or self.status.url == "" or self.status.url.lower() == "None": - if self.spec is not None and self.spec.ingress is not None: - scheme = "https" if self.spec.ingress.tlsSecret is not None else "http" - url = urljoin(f"{scheme}://{self.spec.ingress.host}", self.spec.session.urlPath) - else: - url = self.status.url + if self.base_url is not None: + url = self.base_url ready_containers = 0 total_containers = 0 if self.status.initContainerCounts is not None: @@ -215,3 +211,19 @@ def as_apispec(self) -> apispec.SessionResponse: launcher_id=str(self.launcher_id), resource_class_id=self.resource_class_id, ) + + @property + def base_url(self) -> str | None: + """Get the URL of the session, excluding the default URL from the session launcher.""" + if self.status.url and len(self.status.url) > 0: + return self.status.url + if self.spec is None or self.spec.ingress is None: + return None + scheme = "https" if self.spec and self.spec.ingress and self.spec.ingress.tlsSecret else "http" + host = self.spec.ingress.host + path = self.spec.session.urlPath if self.spec.session.urlPath else "/" + params = None + query = None + fragment = None + url = urlunparse((scheme, host, path, params, query, fragment)) + return url diff --git a/poetry.lock b/poetry.lock index 37ba8f6de..4bd733d51 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1595,6 +1595,23 @@ httpcore = ">=1.0.4" httpx = ">=0.23.1" wsproto = "*" +[[package]] +name = "httpx-ws" +version = "0.6.0" +description = "WebSockets support for HTTPX" +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx_ws-0.6.0-py3-none-any.whl", hash = "sha256:437cfca94519a4e6ae06eb5573192df6c0da85c22b1a19cc1ea0b02b05a51d25"}, + {file = "httpx_ws-0.6.0.tar.gz", hash = "sha256:60218f531fb474a2143af38568f4b7d94ba356780973443365c8e2c87882bb8c"}, +] + +[package.dependencies] +anyio = ">=4" +httpcore = ">=1.0.4" +httpx = ">=0.23.1" +wsproto = "*" + [[package]] name = "hypothesis" version = "6.111.1" diff --git a/projects/background_jobs/poetry.lock b/projects/background_jobs/poetry.lock index 4f19f1ac3..b63577bbd 100644 --- a/projects/background_jobs/poetry.lock +++ b/projects/background_jobs/poetry.lock @@ -92,6 +92,20 @@ files = [ [package.dependencies] cachetools = ">=5.2.0,<6.0.0" +[[package]] +name = "asyncache" +version = "0.3.1" +description = "Helpers to use cachetools with async code." +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "asyncache-0.3.1-py3-none-any.whl", hash = "sha256:ef20a1024d265090dd1e0785c961cf98b9c32cc7d9478973dcf25ac1b80011f5"}, + {file = "asyncache-0.3.1.tar.gz", hash = "sha256:9a1e60a75668e794657489bdea6540ee7e3259c483517b934670db7600bf5035"}, +] + +[package.dependencies] +cachetools = ">=5.2.0,<6.0.0" + [[package]] name = "asyncpg" version = "0.29.0" @@ -992,6 +1006,23 @@ httpcore = ">=1.0.4" httpx = ">=0.23.1" wsproto = "*" +[[package]] +name = "httpx-ws" +version = "0.6.0" +description = "WebSockets support for HTTPX" +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx_ws-0.6.0-py3-none-any.whl", hash = "sha256:437cfca94519a4e6ae06eb5573192df6c0da85c22b1a19cc1ea0b02b05a51d25"}, + {file = "httpx_ws-0.6.0.tar.gz", hash = "sha256:60218f531fb474a2143af38568f4b7d94ba356780973443365c8e2c87882bb8c"}, +] + +[package.dependencies] +anyio = ">=4" +httpcore = ">=1.0.4" +httpx = ">=0.23.1" +wsproto = "*" + [[package]] name = "idna" version = "3.7" diff --git a/projects/renku_data_service/poetry.lock b/projects/renku_data_service/poetry.lock index 9961a448d..809edf0bd 100644 --- a/projects/renku_data_service/poetry.lock +++ b/projects/renku_data_service/poetry.lock @@ -17,6 +17,23 @@ caio = ">=0.9.0,<0.10.0" [package.extras] develop = ["aiomisc-pytest", "coveralls", "pytest", "pytest-cov", "pytest-rst"] +[[package]] +name = "aiofile" +version = "3.8.8" +description = "Asynchronous file operations." +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "aiofile-3.8.8-py3-none-any.whl", hash = "sha256:41e8845cce055779cd77713d949a339deb012eab605b857765e8f8e52a5ed811"}, + {file = "aiofile-3.8.8.tar.gz", hash = "sha256:41f3dc40bd730459d58610476e82e5efb2f84ae6e9fa088a9545385d838b8a43"}, +] + +[package.dependencies] +caio = ">=0.9.0,<0.10.0" + +[package.extras] +develop = ["aiomisc-pytest", "coveralls", "pytest", "pytest-cov", "pytest-rst"] + [[package]] name = "aiofiles" version = "24.1.0" @@ -1364,6 +1381,23 @@ httpcore = ">=1.0.4" httpx = ">=0.23.1" wsproto = "*" +[[package]] +name = "httpx-ws" +version = "0.6.0" +description = "WebSockets support for HTTPX" +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx_ws-0.6.0-py3-none-any.whl", hash = "sha256:437cfca94519a4e6ae06eb5573192df6c0da85c22b1a19cc1ea0b02b05a51d25"}, + {file = "httpx_ws-0.6.0.tar.gz", hash = "sha256:60218f531fb474a2143af38568f4b7d94ba356780973443365c8e2c87882bb8c"}, +] + +[package.dependencies] +anyio = ">=4" +httpcore = ">=1.0.4" +httpx = ">=0.23.1" +wsproto = "*" + [[package]] name = "idna" version = "3.7" diff --git a/projects/secrets_storage/poetry.lock b/projects/secrets_storage/poetry.lock index 1cc69c4b8..f34d480e0 100644 --- a/projects/secrets_storage/poetry.lock +++ b/projects/secrets_storage/poetry.lock @@ -1112,6 +1112,23 @@ httpcore = ">=1.0.4" httpx = ">=0.23.1" wsproto = "*" +[[package]] +name = "httpx-ws" +version = "0.6.0" +description = "WebSockets support for HTTPX" +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx_ws-0.6.0-py3-none-any.whl", hash = "sha256:437cfca94519a4e6ae06eb5573192df6c0da85c22b1a19cc1ea0b02b05a51d25"}, + {file = "httpx_ws-0.6.0.tar.gz", hash = "sha256:60218f531fb474a2143af38568f4b7d94ba356780973443365c8e2c87882bb8c"}, +] + +[package.dependencies] +anyio = ">=4" +httpcore = ">=1.0.4" +httpx = ">=0.23.1" +wsproto = "*" + [[package]] name = "idna" version = "3.7"