Merge branch 'develop' into fix-distutils-warning

.gitignore

@@ -56,6 +56,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+.hypothesis
 
 # Translations
 *.mo

CHANGES.rst

@@ -18,6 +18,19 @@
 Changes
 =======
 
+`2.8.1 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.8.0...v2.8.1>`__ (2023-12-18)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **core:** adapt to Zenodo jsonld changes; send referrer on Zenodo request
+   (`#3643 <https://github.com/SwissDataScienceCenter/renku-python/issues/3643>`__)
+   (`41816d8 <https://github.com/SwissDataScienceCenter/renku-python/commit/41816d8178af0b057cca5bdd91330fbb265f9c03>`__)
+-  **service:** dataset import error
+   (`#3670 <https://github.com/SwissDataScienceCenter/renku-python/issues/3670>`__)
+   (`1795cde <https://github.com/SwissDataScienceCenter/renku-python/commit/1795cde3e1a0cd849618175251f8feab997fecfc>`__)
+
 `2.8.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.7.0...v2.8.0>`__ (2023-11-17)
 -------------------------------------------------------------------------------------------------------
 

helm-chart/renku-core/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 2.8.0
+version: 2.8.1

helm-chart/renku-core/values.yaml

@@ -8,4 +8,4 @@ global:
 versions:
   latest:
     image:
-      tag: v2.8.0
+      tag: v2.8.1

renku/core/dataset/providers/zenodo.py

@@ -179,7 +179,7 @@ class ZenodoDatasetSchema(ProviderDatasetSchema):
             """Schema for Zenodo datasets."""
 
             @pre_load
-            def fix_data(self, data, **kwargs):
+            def fix_data(self, data, **_):
                 """Fix data that is received from Zenodo."""
                 # Fix context
                 context = data.get("@context")
@@ -236,7 +236,7 @@ def fix_data(self, data, **kwargs):
 class ZenodoFileSerializer:
     """Zenodo record file."""
 
-    def __init__(self, *, id=None, checksum=None, links=None, key=None, size=None, **kwargs):
+    def __init__(self, *, id=None, checksum=None, links=None, key=None, size=None, **_):
         self.id = id
         self.checksum = checksum
         self.links = links

renku/core/errors.py

@@ -240,8 +240,8 @@ def __init__(self):
         msg = (
             "There are not any detected outputs in the repository. This can be due to your command not creating "
             "any new files or due to files that get created already existing before the command was run. In the "
-            "latter case, you can remove those files prior to running your command.\nIf you want to track the command"
-            "without outputs, use the use --no-output option.\nYou can also use the --output flag to track outputs"
+            "latter case, you can remove those files prior to running your command.\nIf you want to track the command "
+            "without outputs, use the use --no-output option.\nYou can also use the --output flag to track outputs "
             "manually."
         )
 

renku/core/session/session.py

@@ -179,9 +179,11 @@ def session_start(
     if image_name is None:
         tag = project_context.repository.head.commit.hexsha[:7]
         repo_host = get_image_repository_host()
-        image_name = f"{project_name}:{tag}"
+        image_name = f"{project_name.lower()}:{tag}"
         if repo_host:
             image_name = f"{repo_host}/{image_name}"
+    if image_name.lower() != image_name:
+        raise errors.SessionStartError(f"Image name '{image_name}' cannot contain upper-case letters.")
 
     force_build_image = provider_api.force_build_image(**kwargs)
 

renku/ui/service/controllers/datasets_import.py

@@ -48,7 +48,7 @@ def renku_op(self):
         """Renku operation for the controller."""
         job = self.cache.make_job(
             self.user,
-            # NOTE: To support operation to be execute on remote project, this behaviour should be updated.
+            # NOTE: To support operation to be executed on remote project, this behaviour should be updated.
             project=self.ctx["project_id"],
             job_data={"renku_op": "dataset_import", "client_extras": self.ctx.get("client_extras")},
         )

renku/ui/service/controllers/version.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Renku service version controller."""
+import os
+
 from renku import __version__
 from renku.core.migration.migrate import SUPPORTED_PROJECT_VERSION
 from renku.ui.service.controllers.api.abstract import ServiceCtrl
@@ -33,6 +35,7 @@ def to_response(self, minimum_version, maximum_version):
             {
                 "latest_version": __version__,
                 "supported_project_version": SUPPORTED_PROJECT_VERSION,
+                "cli_version": os.environ.get("RENKU_PROJECT_DEFAULT_CLI_VERSION") or __version__,
                 "minimum_api_version": minimum_version.name,
                 "maximum_api_version": maximum_version.name,
             },

renku/ui/service/errors.py

@@ -121,7 +121,7 @@ def _handle_sentry(self):
                         sentry_target = sentry_url.netloc.split("@")[-1]
                         # NOTE: sentry doesn't support a global search. A proper link would require the specific org
                         sentry = f"{sentry_url.scheme }://{sentry_target}/organizations/sentry?query={sentry}"
-                    except Exception:
+                    except Exception:  # nosec
                         pass
             except KeyError as e:
                 sentry = f"Unexpected error while reporting to Sentry: {str(e)}"
@@ -348,6 +348,19 @@ def __init__(self, exception=None):
         super().__init__(exception=exception)
 
 
+class UserDatasetsNotFoundError(ServiceError):
+    """Dataset couldn't be found in project."""
+
+    code = SVC_ERROR_USER + 133
+    userMessage = (
+        "The dataset doesn't exist in the project. Please check your inputs or create the target dataset first."
+    )
+    devMessage = "The dataset couldn't be found in the project."
+
+    def __init__(self, exception=None):
+        super().__init__(exception=exception)
+
+
 class UserOutdatedProjectError(ServiceError):
     """The operation can be done only after updating the target project."""
 

renku/ui/service/jobs/datasets.py

@@ -37,7 +37,7 @@ def dataset_import(
     user_job_id,
     project_id,
     dataset_uri,
-    name=None,
+    slug=None,
     extract=False,
     tag=None,
     timeout=None,
@@ -63,7 +63,7 @@ def dataset_import(
             command = import_dataset_command().with_commit_message(commit_message)
             command.with_communicator(communicator).build().execute(
                 uri=dataset_uri,
-                name=name,
+                slug=slug,
                 extract=extract,
                 tag=tag,
                 yes=True,
@@ -98,7 +98,7 @@ def _is_safe_to_pass_gitlab_token(project_git_url, dataset_uri):
 
 
 @requires_cache
-def dataset_add_remote_file(cache, user, user_job_id, project_id, create_dataset, commit_message, name, url):
+def dataset_add_remote_file(cache, user, user_job_id, project_id, create_dataset, commit_message, slug, url):
     """Add a remote file to a specified dataset."""
     user = cache.ensure_user(user)
     worker_log.debug(f"executing dataset add remote file job for {user.user_id}:{user.fullname}")
@@ -113,9 +113,9 @@ def dataset_add_remote_file(cache, user, user_job_id, project_id, create_dataset
         with renku_project_context(project.abs_path):
             urls = url if isinstance(url, list) else [url]
 
-            worker_log.debug(f"adding files {urls} to dataset {name}")
+            worker_log.debug(f"adding files {urls} to dataset {slug}")
             command = add_to_dataset_command().with_commit_message(commit_message).build()
-            result = command.execute(dataset_slug=name, urls=urls, create=create_dataset)
+            result = command.execute(dataset_slug=slug, urls=urls, create=create_dataset)
             if result.error:
                 raise result.error
 

renku/ui/service/views/error_handlers.py

@@ -28,6 +28,7 @@
     AuthenticationError,
     DatasetExistsError,
     DatasetImageError,
+    DatasetNotFound,
     DockerfileUpdateError,
     GitCommandError,
     GitError,
@@ -65,6 +66,7 @@
     ProgramUpdateProjectError,
     ServiceError,
     UserDatasetsMultipleImagesError,
+    UserDatasetsNotFoundError,
     UserDatasetsUnlinkError,
     UserDatasetsUnreachableImageError,
     UserInvalidGenericFieldsError,
@@ -368,6 +370,8 @@ def decorated_function(*args, **kwargs):
                 if "".join(value) == "Field may not be null.":
                     raise UserMissingFieldError(e, key)
             raise
+        except DatasetNotFound as e:
+            raise UserDatasetsNotFoundError(e)
         except DatasetExistsError as e:
             raise IntermittentDatasetExistsError(e)
         except RenkuException as e:

tests/service/jobs/test_datasets.py

@@ -88,8 +88,8 @@ def test_dataset_url_import_job(url, svc_client_with_repo):
 @pytest.mark.service
 @retry_failed
 @pytest.mark.vcr
-def test_dataset_import_job(doi, svc_client_with_repo):
-    """Test dataset import via doi."""
+def test_dataset_import_job_with_slug(doi, svc_client_with_repo):
+    """Test dataset import via doi and give it a slug."""
     svc_client, headers, project_id, url_components = svc_client_with_repo
 
     user_id = encode_b64(secure_filename("9ab2fc80-3a5c-426d-ae78-56de01d214df"))
@@ -118,7 +118,14 @@ def test_dataset_import_job(doi, svc_client_with_repo):
     job_id = response.json["result"]["job_id"]
 
     commit_message = "service: import remote dataset"
-    dataset_import(user, job_id, project_id, doi, commit_message=commit_message)
+    dataset_import(
+        user=user,
+        user_job_id=job_id,
+        project_id=project_id,
+        dataset_uri=doi,
+        slug="dataset-slug",
+        commit_message=commit_message,
+    )
 
     new_commit = Repository(dest).head.commit
     assert old_commit.hexsha != new_commit.hexsha
@@ -279,7 +286,16 @@ def test_dataset_add_remote_file(url, svc_client_with_repo):
     job_id = response.json["result"]["files"][0]["job_id"]
     commit_message = "service: dataset add remote file"
 
-    dataset_add_remote_file(user, job_id, project_id, True, commit_message, payload["slug"], url)
+    # user, user_job_id, project_id, create_dataset, commit_message, slug, url
+    dataset_add_remote_file(
+        user=user,
+        user_job_id=job_id,
+        project_id=project_id,
+        create_dataset=True,
+        commit_message=commit_message,
+        slug=payload["slug"],
+        url=url,
+    )
 
     new_commit = Repository(dest).head.commit