From 72a123f2ea07e432ca692fa8fdd2f44e1042c7d1 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Wed, 22 Jan 2025 15:30:50 +0100 Subject: [PATCH 01/37] feature(study): wip --- antarest/study/service.py | 4 ++-- tests/storage/test_service.py | 38 ++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/antarest/study/service.py b/antarest/study/service.py index e0e1e1a9c2..e7f0c39e3d 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -978,12 +978,12 @@ def sync_studies_on_disk( study_paths = [study.path for study in all_studies if study.missing is None] missing_studies = {study.path: study for study in all_studies if study.missing is not None} for folder in folders: - study_path = str(folder.path) + study_path = str(folder.path.as_posix()) if study_path not in study_paths: try: if study_path not in missing_studies.keys(): base_path = self.config.storage.workspaces[folder.workspace].path - dir_name = folder.path.relative_to(base_path) + dir_name = folder.path.relative_to(base_path).as_posix() study = RawStudy( id=str(uuid4()), name=folder.path.name, diff --git a/tests/storage/test_service.py b/tests/storage/test_service.py index b31ee0ea7f..11630b1ef9 100644 --- a/tests/storage/test_service.py +++ b/tests/storage/test_service.py @@ -17,7 +17,7 @@ import uuid from configparser import MissingSectionHeaderError from datetime import datetime, timedelta, timezone -from pathlib import Path +from pathlib import Path, PureWindowsPath from unittest.mock import ANY, Mock, call, patch, seal import pytest @@ -362,6 +362,42 @@ def test_partial_sync_studies_from_disk() -> None: ) +# noinspection PyArgumentList +@pytest.mark.unit_test +def test_windows_path_scan() -> None: + """ + In this test scenarion only study a exist in database, we call the sync with a folder containing study b. + + The input folder is a windows folder. + + We expect the sync to save the study b with a posix path and not a windows path. + + Note : study a won't get deleted, the scan is partial as we give a directory argument. + """ + ma = RawStudy(id="a", path="a") + fb = StudyFolder(path=PureWindowsPath("directory\\b"), workspace=DEFAULT_WORKSPACE_NAME, groups=[]) + + repository = Mock() + repository.get_all_raw.side_effect = [[ma]] + config = Config(storage=StorageConfig(workspaces={DEFAULT_WORKSPACE_NAME: WorkspaceConfig()})) + service = build_study_service(Mock(), repository, config) + + service.sync_studies_on_disk([fb], directory=PureWindowsPath("directory")) + + repository.save.assert_called_with( + RawStudy( + id=ANY, + path="directory/b", + name="b", + folder="directory/b", + created_at=ANY, + missing=None, + public_mode=PublicMode.FULL, + workspace=DEFAULT_WORKSPACE_NAME, + ) + ) + + @with_db_context def test_remove_duplicate(db_session: Session) -> None: with db_session: From 8220f66462166bb1fbd4ac8a1458d258c5ad5aa2 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Thu, 23 Jan 2025 17:22:19 +0100 Subject: [PATCH 02/37] feature(study): normalize path using the slqlachemy property decorator --- antarest/study/model.py | 15 +++++++++++++-- antarest/study/service.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index d2ff8b97b9..ca573c7634 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -16,7 +16,7 @@ import typing as t import uuid from datetime import datetime, timedelta -from pathlib import Path +from pathlib import Path, PureWindowsPath from antares.study.version import StudyVersion from pydantic import BeforeValidator, ConfigDict, Field, PlainSerializer, computed_field, field_validator @@ -230,7 +230,7 @@ class Study(Base): # type: ignore created_at = Column(DateTime, index=True) updated_at = Column(DateTime, index=True) last_access = Column(DateTime) - path = Column(String()) + _path = Column(String()) folder = Column(String, nullable=True, index=True) parent_id = Column(String(36), ForeignKey("study.id", name="fk_study_study_id"), index=True) public_mode = Column(Enum(PublicMode), default=PublicMode.NONE) @@ -288,6 +288,17 @@ def __eq__(self, other: t.Any) -> bool: def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} + @property + def path(self): + return self._path + + @path.setter + def path(self, path: str): + windows_path = PureWindowsPath(path) + posix_path = windows_path.as_posix() + path_str = str(posix_path) + self._path = path_str + class RawStudy(Study): """ diff --git a/antarest/study/service.py b/antarest/study/service.py index e7f0c39e3d..b4d753fd4e 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -983,7 +983,7 @@ def sync_studies_on_disk( try: if study_path not in missing_studies.keys(): base_path = self.config.storage.workspaces[folder.workspace].path - dir_name = folder.path.relative_to(base_path).as_posix() + dir_name = folder.path.relative_to(base_path) study = RawStudy( id=str(uuid4()), name=folder.path.name, From 282fcd664159da8ee6af9395a522e618bca3af56 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Fri, 24 Jan 2025 10:20:26 +0100 Subject: [PATCH 03/37] feature(study): normalize folder path using sqlachemy property --- antarest/study/model.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index ca573c7634..4d4fa37900 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -230,8 +230,8 @@ class Study(Base): # type: ignore created_at = Column(DateTime, index=True) updated_at = Column(DateTime, index=True) last_access = Column(DateTime) - _path = Column(String()) - folder = Column(String, nullable=True, index=True) + _path = Column("path", String) + _folder = Column("folder", String, nullable=True, index=True) parent_id = Column(String(36), ForeignKey("study.id", name="fk_study_study_id"), index=True) public_mode = Column(Enum(PublicMode), default=PublicMode.NONE) owner_id = Column(Integer, ForeignKey(Identity.id), nullable=True, index=True) @@ -294,10 +294,22 @@ def path(self): @path.setter def path(self, path: str): - windows_path = PureWindowsPath(path) - posix_path = windows_path.as_posix() - path_str = str(posix_path) - self._path = path_str + self._path = normalize_path(path) + + @property + def folder(self): + return self._folder + + @folder.setter + def folder(self, folder: str): + self._folder = normalize_path(folder) + + +def normalize_path(path: str) -> str: + if not path: + return path + windows_path = PureWindowsPath(path) + return windows_path.as_posix() class RawStudy(Study): From 021df2e68a2a3947d5311034eeae0c04325d4b8b Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Fri, 24 Jan 2025 15:49:14 +0100 Subject: [PATCH 04/37] fix(studies): use hybrid property instead of property --- antarest/study/model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index 4d4fa37900..3db617f829 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -30,6 +30,7 @@ PrimaryKeyConstraint, String, ) +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship # type: ignore from typing_extensions import override @@ -288,7 +289,7 @@ def __eq__(self, other: t.Any) -> bool: def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} - @property + @hybrid_property def path(self): return self._path @@ -296,7 +297,7 @@ def path(self): def path(self, path: str): self._path = normalize_path(path) - @property + @hybrid_property def folder(self): return self._folder From 9275c03c53480ad24abaa54ea4a5b0c4e2aa414b Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Mon, 27 Jan 2025 10:41:58 +0100 Subject: [PATCH 05/37] fix(study): fix test and typing --- antarest/study/model.py | 10 +++++++--- antarest/study/repository.py | 2 +- antarest/study/service.py | 2 +- antarest/study/storage/rawstudy/raw_study_service.py | 4 ++-- tests/storage/business/test_import.py | 4 ++-- tests/storage/business/test_raw_study_service.py | 6 +----- tests/storage/test_service.py | 11 +++++------ 7 files changed, 19 insertions(+), 20 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index 3db617f829..fa371134bc 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -30,7 +30,7 @@ PrimaryKeyConstraint, String, ) -from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.ext.hybrid import hybrid_property # type: ignore from sqlalchemy.orm import relationship # type: ignore from typing_extensions import override @@ -289,11 +289,12 @@ def __eq__(self, other: t.Any) -> bool: def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} + # must use hybrid property instead of property to be able to use it in query @hybrid_property def path(self): return self._path - @path.setter + @path.setter # type:ignore def path(self, path: str): self._path = normalize_path(path) @@ -301,12 +302,15 @@ def path(self, path: str): def folder(self): return self._folder - @folder.setter + @folder.setter # type:ignore def folder(self, folder: str): self._folder = normalize_path(folder) def normalize_path(path: str) -> str: + """ + We want to store the path in posix format in the database, even on windows. + """ if not path: return path windows_path = PureWindowsPath(path) diff --git a/antarest/study/repository.py b/antarest/study/repository.py index a904124d4d..216bd58226 100644 --- a/antarest/study/repository.py +++ b/antarest/study/repository.py @@ -427,7 +427,7 @@ def list_duplicates(self) -> t.List[t.Tuple[str, str]]: """ session = self.session subquery = session.query(Study.path).group_by(Study.path).having(func.count() > 1).subquery() - query = session.query(Study.id, Study.path).filter(Study.path.in_(subquery)) + query = session.query(Study.id, Study.path).filter(Study.path.in_(subquery)) # type:ignore return t.cast(t.List[t.Tuple[str, str]], query.all()) def has_children(self, uuid: str) -> bool: diff --git a/antarest/study/service.py b/antarest/study/service.py index 9b7d259e94..1df5db4e75 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1125,7 +1125,7 @@ def move_study(self, study_id: str, folder_dest: str, params: RequestParameters) new_folder = folder_dest.rstrip("/") + f"/{study.id}" else: new_folder = None - study.folder = new_folder + study.folder = new_folder # type:ignore self.repository.save(study, update_modification_date=False) self.event_bus.push( Event( diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index 4d2a027965..f756ed1342 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -217,7 +217,7 @@ def create(self, metadata: RawStudy) -> RawStudy: study = self.study_factory.create_from_fs(path_study, metadata.id) update_antares_info(metadata, study.tree, update_author=True) - metadata.path = str(path_study) + metadata.path = str(path_study) # type:ignore return metadata @@ -343,7 +343,7 @@ def import_study(self, metadata: RawStudy, stream: t.BinaryIO) -> Study: shutil.rmtree(study_path) raise - metadata.path = str(study_path) + metadata.path = str(study_path) # type:ignore return metadata @override diff --git a/tests/storage/business/test_import.py b/tests/storage/business/test_import.py index 7b8af36f65..3b382f23ae 100644 --- a/tests/storage/business/test_import.py +++ b/tests/storage/business/test_import.py @@ -68,7 +68,7 @@ def test_import_study(tmp_path: Path) -> None: ) with path_zip.open("rb") as input_file: md = study_service.import_study(md, input_file) - assert md.path == f"{tmp_path}{os.sep}other-study-zip" + assert md.path == f"{tmp_path}/other-study-zip" # assert that importing file into a created study does not alter its group assert md.groups == ["fake_group_1", "fake_group_2"] @@ -95,7 +95,7 @@ def test_import_study(tmp_path: Path) -> None: ) with filepath_7zip.open("rb") as input_file: md = study_service.import_study(md, input_file) - assert md.path == f"{tmp_path}{os.sep}other-study-7zip" + assert md.path == f"{tmp_path}/other-study-7zip" # assert that importing file into a created study does not alter its group assert md.groups == ["fake_group_1", "fake_group_2"] diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index eb87bfc09f..f963206b00 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -66,9 +66,6 @@ def test_get(tmp_path: str, project_path) -> None: data = {"titi": 43} sub_route = "settings" - path = path_study / "settings" - key = "titi" - study = Mock() study.get.return_value = data study_factory = Mock() @@ -100,7 +97,6 @@ def test_get_cache(tmp_path: str) -> None: path_study.mkdir() (path_study / "settings").mkdir() (path_study / "study.antares").touch() - path = path_study / "settings" data = {"titi": 43} study = Mock() @@ -457,7 +453,7 @@ def test_copy_study( ) md = study_service.copy(src_md, "dst_name", groups) md_id = md.id - assert str(md.path) == f"{tmp_path}{os.sep}{md_id}" + assert str(md.path) == f"{tmp_path}/{md_id}" assert md.public_mode == PublicMode.NONE assert md.groups == groups study.get.assert_called_once_with(["study"]) diff --git a/tests/storage/test_service.py b/tests/storage/test_service.py index 11630b1ef9..637fa02842 100644 --- a/tests/storage/test_service.py +++ b/tests/storage/test_service.py @@ -11,7 +11,6 @@ # This file is part of the Antares project. import contextlib -import os import textwrap import typing as t import uuid @@ -319,7 +318,7 @@ def test_partial_sync_studies_from_disk() -> None: mb = RawStudy(id="b", path="b") mc = RawStudy( id="c", - path=f"directory{os.sep}c", + path="directory/c", name="c", content_status=StudyContentStatus.WARNING, workspace=DEFAULT_WORKSPACE_NAME, @@ -327,12 +326,12 @@ def test_partial_sync_studies_from_disk() -> None: ) md = RawStudy( id="d", - path=f"directory{os.sep}d", + path="directory/d", missing=datetime.utcnow() - timedelta(MAX_MISSING_STUDY_TIMEOUT + 1), ) me = RawStudy( id="e", - path=f"directory{os.sep}e", + path="directory/e", created_at=now, missing=datetime.utcnow() - timedelta(MAX_MISSING_STUDY_TIMEOUT - 1), ) @@ -351,9 +350,9 @@ def test_partial_sync_studies_from_disk() -> None: repository.save.assert_called_with( RawStudy( id=ANY, - path=f"directory{os.sep}f", + path="directory/f", name="f", - folder=f"directory{os.sep}f", + folder="directory/f", created_at=ANY, missing=None, public_mode=PublicMode.FULL, From 390887ebaaca16a1d7519d34260ecd59c4bb447e Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Mon, 27 Jan 2025 16:14:40 +0100 Subject: [PATCH 06/37] fix(study): use validate instead of property --- antarest/study/model.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index fa371134bc..3e63994ae2 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -30,8 +30,8 @@ PrimaryKeyConstraint, String, ) -from sqlalchemy.ext.hybrid import hybrid_property # type: ignore from sqlalchemy.orm import relationship # type: ignore +from sqlalchemy.orm import validates from typing_extensions import override from antarest.core.exceptions import ShouldNotHappenException @@ -231,8 +231,8 @@ class Study(Base): # type: ignore created_at = Column(DateTime, index=True) updated_at = Column(DateTime, index=True) last_access = Column(DateTime) - _path = Column("path", String) - _folder = Column("folder", String, nullable=True, index=True) + path = Column(String) + folder = Column(String, nullable=True, index=True) parent_id = Column(String(36), ForeignKey("study.id", name="fk_study_study_id"), index=True) public_mode = Column(Enum(PublicMode), default=PublicMode.NONE) owner_id = Column(Integer, ForeignKey(Identity.id), nullable=True, index=True) @@ -289,22 +289,13 @@ def __eq__(self, other: t.Any) -> bool: def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} - # must use hybrid property instead of property to be able to use it in query - @hybrid_property - def path(self): - return self._path + @validates("path") + def validate_email(self, key, path): + return normalize_path(path) - @path.setter # type:ignore - def path(self, path: str): - self._path = normalize_path(path) - - @hybrid_property - def folder(self): - return self._folder - - @folder.setter # type:ignore - def folder(self, folder: str): - self._folder = normalize_path(folder) + @validates("folder") + def validate_folder(self, key, folder): + return normalize_path(folder) def normalize_path(path: str) -> str: From 2b78876fbb2a4027bd818a8b6c1a3d904457e1c4 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Mon, 27 Jan 2025 16:14:59 +0100 Subject: [PATCH 07/37] fix(study): fix tests on windows --- tests/storage/business/test_import.py | 9 +++++---- tests/storage/business/test_raw_study_service.py | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/storage/business/test_import.py b/tests/storage/business/test_import.py index 3b382f23ae..8214698f5e 100644 --- a/tests/storage/business/test_import.py +++ b/tests/storage/business/test_import.py @@ -51,8 +51,9 @@ def test_import_study(tmp_path: Path) -> None: study_service.get_study_path = Mock() study_service.get.return_value = data + tmp_path_posix = Path(tmp_path.as_posix()) # first test importing a study for an archived study with `.zip` format - study_service.get_study_path.return_value = tmp_path / "other-study-zip" + study_service.get_study_path.return_value = tmp_path_posix / "other-study-zip" filepath_zip = shutil.make_archive(str(study_path.absolute()), "zip", study_path) shutil.rmtree(study_path) @@ -68,11 +69,11 @@ def test_import_study(tmp_path: Path) -> None: ) with path_zip.open("rb") as input_file: md = study_service.import_study(md, input_file) - assert md.path == f"{tmp_path}/other-study-zip" + assert md.path == f"{tmp_path_posix}/other-study-zip" # assert that importing file into a created study does not alter its group assert md.groups == ["fake_group_1", "fake_group_2"] - shutil.rmtree(tmp_path / "other-study-zip") + shutil.rmtree(tmp_path_posix / "other-study-zip") # second test for an archived study with a `.7z` format study_service.get_study_path.return_value = tmp_path / "other-study-7zip" @@ -95,7 +96,7 @@ def test_import_study(tmp_path: Path) -> None: ) with filepath_7zip.open("rb") as input_file: md = study_service.import_study(md, input_file) - assert md.path == f"{tmp_path}/other-study-7zip" + assert md.path == f"{tmp_path_posix}/other-study-7zip" # assert that importing file into a created study does not alter its group assert md.groups == ["fake_group_1", "fake_group_2"] diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index f963206b00..b2284a2072 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -233,7 +233,8 @@ def test_create(tmp_path: Path, project_path: Path) -> None: ) md = study_service.create(metadata) - assert md.path == str(tmp_path / "study1") + tmp_path_posix = tmp_path.as_posix() + assert md.path == str(tmp_path_posix / "study1") path_study = tmp_path / md.id assert path_study.exists() @@ -453,7 +454,8 @@ def test_copy_study( ) md = study_service.copy(src_md, "dst_name", groups) md_id = md.id - assert str(md.path) == f"{tmp_path}/{md_id}" + tmp_path_as_posix = tmp_path.as_posix() + assert str(md.path) == f"{tmp_path_as_posix}/{md_id}" assert md.public_mode == PublicMode.NONE assert md.groups == groups study.get.assert_called_once_with(["study"]) From e764c224da6cfbea6f8f726ed3fd0f745dbe9a8e Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Mon, 27 Jan 2025 16:28:24 +0100 Subject: [PATCH 08/37] fix(build): fix test --- tests/storage/business/test_raw_study_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index b2284a2072..6874817656 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -233,7 +233,7 @@ def test_create(tmp_path: Path, project_path: Path) -> None: ) md = study_service.create(metadata) - tmp_path_posix = tmp_path.as_posix() + tmp_path_posix = Path(tmp_path.as_posix()) assert md.path == str(tmp_path_posix / "study1") path_study = tmp_path / md.id assert path_study.exists() From f05997025d3485c055f7d6282701530f9abf7109 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Mon, 27 Jan 2025 16:57:57 +0100 Subject: [PATCH 09/37] fix(build): fix types --- antarest/study/model.py | 4 ++-- antarest/study/repository.py | 2 +- antarest/study/service.py | 2 +- antarest/study/storage/rawstudy/raw_study_service.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index 3e63994ae2..968800d31c 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -290,11 +290,11 @@ def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} @validates("path") - def validate_email(self, key, path): + def validate_email(self, key, path) -> str: return normalize_path(path) @validates("folder") - def validate_folder(self, key, folder): + def validate_folder(self, key, folder) -> str: return normalize_path(folder) diff --git a/antarest/study/repository.py b/antarest/study/repository.py index 216bd58226..a904124d4d 100644 --- a/antarest/study/repository.py +++ b/antarest/study/repository.py @@ -427,7 +427,7 @@ def list_duplicates(self) -> t.List[t.Tuple[str, str]]: """ session = self.session subquery = session.query(Study.path).group_by(Study.path).having(func.count() > 1).subquery() - query = session.query(Study.id, Study.path).filter(Study.path.in_(subquery)) # type:ignore + query = session.query(Study.id, Study.path).filter(Study.path.in_(subquery)) return t.cast(t.List[t.Tuple[str, str]], query.all()) def has_children(self, uuid: str) -> bool: diff --git a/antarest/study/service.py b/antarest/study/service.py index 1df5db4e75..9b7d259e94 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1125,7 +1125,7 @@ def move_study(self, study_id: str, folder_dest: str, params: RequestParameters) new_folder = folder_dest.rstrip("/") + f"/{study.id}" else: new_folder = None - study.folder = new_folder # type:ignore + study.folder = new_folder self.repository.save(study, update_modification_date=False) self.event_bus.push( Event( diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index f756ed1342..4d2a027965 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -217,7 +217,7 @@ def create(self, metadata: RawStudy) -> RawStudy: study = self.study_factory.create_from_fs(path_study, metadata.id) update_antares_info(metadata, study.tree, update_author=True) - metadata.path = str(path_study) # type:ignore + metadata.path = str(path_study) return metadata @@ -343,7 +343,7 @@ def import_study(self, metadata: RawStudy, stream: t.BinaryIO) -> Study: shutil.rmtree(study_path) raise - metadata.path = str(study_path) # type:ignore + metadata.path = str(study_path) return metadata @override From d9cb6d0f40623e9bc2b3d3e1e8ec8d3d7746c38b Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Mon, 27 Jan 2025 17:37:28 +0100 Subject: [PATCH 10/37] fix(build): fix tests --- tests/storage/business/test_import.py | 13 ++++++------- tests/storage/business/test_raw_study_service.py | 7 +++---- tests/storage/conftest.py | 7 ++++++- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/storage/business/test_import.py b/tests/storage/business/test_import.py index 8214698f5e..0958b28fde 100644 --- a/tests/storage/business/test_import.py +++ b/tests/storage/business/test_import.py @@ -27,9 +27,9 @@ @pytest.mark.unit_test -def test_import_study(tmp_path: Path) -> None: +def test_import_study(tmp_path_posix: Path) -> None: name = "my-study" - study_path = tmp_path / name + study_path = tmp_path_posix / name study_path.mkdir() (study_path / "study.antares").touch() @@ -51,7 +51,6 @@ def test_import_study(tmp_path: Path) -> None: study_service.get_study_path = Mock() study_service.get.return_value = data - tmp_path_posix = Path(tmp_path.as_posix()) # first test importing a study for an archived study with `.zip` format study_service.get_study_path.return_value = tmp_path_posix / "other-study-zip" @@ -63,7 +62,7 @@ def test_import_study(tmp_path: Path) -> None: md = RawStudy( id="other-study-zip", workspace=DEFAULT_WORKSPACE_NAME, - path=tmp_path / "other-study-zip", + path=tmp_path_posix / "other-study-zip", additional_data=StudyAdditionalData(), groups=["fake_group_1", "fake_group_2"], ) @@ -76,7 +75,7 @@ def test_import_study(tmp_path: Path) -> None: shutil.rmtree(tmp_path_posix / "other-study-zip") # second test for an archived study with a `.7z` format - study_service.get_study_path.return_value = tmp_path / "other-study-7zip" + study_service.get_study_path.return_value = tmp_path_posix / "other-study-7zip" study_path.mkdir() (study_path / "study.antares").touch() @@ -90,7 +89,7 @@ def test_import_study(tmp_path: Path) -> None: md = RawStudy( id="other-study-7zip", workspace=DEFAULT_WORKSPACE_NAME, - path=tmp_path / "other-study-7zip", + path=tmp_path_posix / "other-study-7zip", additional_data=StudyAdditionalData(), groups=["fake_group_1", "fake_group_2"], ) @@ -100,7 +99,7 @@ def test_import_study(tmp_path: Path) -> None: # assert that importing file into a created study does not alter its group assert md.groups == ["fake_group_1", "fake_group_2"] - shutil.rmtree(tmp_path / "other-study-7zip") + shutil.rmtree(tmp_path_posix / "other-study-7zip") # test for an unsupported archive format with pytest.raises(BadArchiveContent, match="Unsupported archive format"): diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 6874817656..9885b33d09 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -206,14 +206,14 @@ def test_assert_study_not_exist(tmp_path: str, project_path) -> None: @pytest.mark.unit_test -def test_create(tmp_path: Path, project_path: Path) -> None: +def test_create(tmp_path_posix: Path, project_path: Path) -> None: study = Mock() data = {"antares": {"caption": None}} study.get.return_value = data study_factory = Mock() study_factory.create_from_fs.return_value = FileStudy(Mock(), study) - config = build_config(tmp_path) + config = build_config(tmp_path_posix) study_service = RawStudyService( config=config, cache=Mock(), @@ -233,9 +233,8 @@ def test_create(tmp_path: Path, project_path: Path) -> None: ) md = study_service.create(metadata) - tmp_path_posix = Path(tmp_path.as_posix()) assert md.path == str(tmp_path_posix / "study1") - path_study = tmp_path / md.id + path_study = tmp_path_posix / md.id assert path_study.exists() path_study_antares_infos = path_study / "study.antares" diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 297a367d08..dd19f75005 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -343,3 +343,8 @@ def get_all(self, owner: Optional[int] = None) -> List[FileDownload]: class SimpleFileTransferManager(FileTransferManager): def __init__(self, config: Config): super().__init__(FileDownloadRepositoryMock(), Mock(), config) + + +@pytest.fixture +def tmp_path_posix(tmp_path: Path) -> Path: + return Path(PureWindowsPath(tmp_path).as_posix()) From 88838309e46f12834c14f70e1ca224940b647047 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 11:14:21 +0100 Subject: [PATCH 11/37] test only on windows --- .github/workflows/main.yml | 220 +++++++++--------- .../business/test_raw_study_service.py | 1 + 2 files changed, 111 insertions(+), 110 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5207a3d20..6c9262bd27 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,40 +5,40 @@ on: - "**" jobs: - python-lint: - runs-on: ubuntu-20.04 - steps: - - name: Checkout github repo (+ download lfs dependencies) - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - uses: isort/isort-action@v1.1.1 - with: - sort-paths: antarest, tests - requirementsFiles: "requirements-dev.txt" - - name: Check with black - uses: psf/black@stable - with: - # Version of Black should match the versions set in `requirements-dev.txt` - version: "~=23.7.0" - options: --check --diff - - name: Check Typing (mypy) - #continue-on-error: true - run: | - mypy --install-types --non-interactive + # python-lint: + # runs-on: ubuntu-20.04 + # steps: + # - name: Checkout github repo (+ download lfs dependencies) + # uses: actions/checkout@v4 + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: 3.11 + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install -r requirements-dev.txt + # - uses: isort/isort-action@v1.1.1 + # with: + # sort-paths: antarest, tests + # requirementsFiles: "requirements-dev.txt" + # - name: Check with black + # uses: psf/black@stable + # with: + # # Version of Black should match the versions set in `requirements-dev.txt` + # version: "~=23.7.0" + # options: --check --diff + # - name: Check Typing (mypy) + # #continue-on-error: true + # run: | + # mypy --install-types --non-interactive python-test: runs-on: ${{ matrix.os }} strategy: max-parallel: 9 matrix: - os: [windows-latest, ubuntu-20.04] + os: [windows-latest] steps: - name: Checkout github repo (+ download lfs dependencies) @@ -53,7 +53,7 @@ jobs: pip install -r requirements-dev.txt - name: Test with pytest run: | - pytest --cov antarest --cov-report xml -n auto --dist=worksteal + pytest --cov antarest --cov-report xml -n auto --dist=worksteal tests/storage/business/test_raw_study_service.py - name: Archive code coverage results if: matrix.os == 'ubuntu-20.04' uses: actions/upload-artifact@v4 @@ -61,90 +61,90 @@ jobs: name: python-code-coverage-report path: coverage.xml - npm-setup: - runs-on: ubuntu-20.04 - steps: - - name: Checkout github repo - uses: actions/checkout@v4 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22.13.0 - - name: Cache node modules - uses: actions/cache@v4 - with: - path: webapp/node_modules - key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-webapp-node-modules- - ${{ runner.os }}- - save-always: true - - name: Install dependencies - run: npm install - working-directory: webapp + # npm-setup: + # runs-on: ubuntu-20.04 + # steps: + # - name: Checkout github repo + # uses: actions/checkout@v4 + # - name: Set up Node.js + # uses: actions/setup-node@v4 + # with: + # node-version: 22.13.0 + # - name: Cache node modules + # uses: actions/cache@v4 + # with: + # path: webapp/node_modules + # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + # restore-keys: | + # ${{ runner.os }}-webapp-node-modules- + # ${{ runner.os }}- + # save-always: true + # - name: Install dependencies + # run: npm install + # working-directory: webapp - npm-lint: - needs: npm-setup - runs-on: ubuntu-20.04 - steps: - - name: Checkout github repo - uses: actions/checkout@v4 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22.13.0 - - name: Restore node modules - uses: actions/cache@v4 - with: - path: webapp/node_modules - key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - save-always: true - - name: Lint - run: npm run lint - working-directory: webapp + # npm-lint: + # needs: npm-setup + # runs-on: ubuntu-20.04 + # steps: + # - name: Checkout github repo + # uses: actions/checkout@v4 + # - name: Set up Node.js + # uses: actions/setup-node@v4 + # with: + # node-version: 22.13.0 + # - name: Restore node modules + # uses: actions/cache@v4 + # with: + # path: webapp/node_modules + # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + # save-always: true + # - name: Lint + # run: npm run lint + # working-directory: webapp - npm-test: - needs: npm-setup - runs-on: ubuntu-20.04 - steps: - - name: Checkout github repo - uses: actions/checkout@v4 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22.13.0 - - name: Restore node modules - uses: actions/cache@v4 - with: - path: webapp/node_modules - key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - save-always: true - - name: Test - run: npm run test - working-directory: webapp + # npm-test: + # needs: npm-setup + # runs-on: ubuntu-20.04 + # steps: + # - name: Checkout github repo + # uses: actions/checkout@v4 + # - name: Set up Node.js + # uses: actions/setup-node@v4 + # with: + # node-version: 22.13.0 + # - name: Restore node modules + # uses: actions/cache@v4 + # with: + # path: webapp/node_modules + # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + # save-always: true + # - name: Test + # run: npm run test + # working-directory: webapp - npm-build: - needs: npm-setup - runs-on: ubuntu-20.04 - steps: - - name: Checkout github repo - uses: actions/checkout@v4 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22.13.0 - - name: Restore node modules - uses: actions/cache@v4 - with: - path: webapp/node_modules - key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - save-always: true - - name: Build - run: npm run build - working-directory: webapp - env: - NODE_OPTIONS: --max-old-space-size=8192 - DISABLE_ESLINT_PLUGIN: true + # npm-build: + # needs: npm-setup + # runs-on: ubuntu-20.04 + # steps: + # - name: Checkout github repo + # uses: actions/checkout@v4 + # - name: Set up Node.js + # uses: actions/setup-node@v4 + # with: + # node-version: 22.13.0 + # - name: Restore node modules + # uses: actions/cache@v4 + # with: + # path: webapp/node_modules + # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + # save-always: true + # - name: Build + # run: npm run build + # working-directory: webapp + # env: + # NODE_OPTIONS: --max-old-space-size=8192 + # DISABLE_ESLINT_PLUGIN: true sonarcloud: runs-on: ubuntu-20.04 diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 9885b33d09..028a7c5b7f 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -233,6 +233,7 @@ def test_create(tmp_path_posix: Path, project_path: Path) -> None: ) md = study_service.create(metadata) + print("tmp_path_posix", tmp_path_posix) assert md.path == str(tmp_path_posix / "study1") path_study = tmp_path_posix / md.id assert path_study.exists() From 7ed7af79b7d393213c8f4bb1e9f5c49055923bbf Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 15:02:14 +0100 Subject: [PATCH 12/37] wip --- .github/workflows/main.yml | 54 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c9262bd27..6f619ed434 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,33 +5,33 @@ on: - "**" jobs: - # python-lint: - # runs-on: ubuntu-20.04 - # steps: - # - name: Checkout github repo (+ download lfs dependencies) - # uses: actions/checkout@v4 - # - name: Set up Python - # uses: actions/setup-python@v5 - # with: - # python-version: 3.11 - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install -r requirements-dev.txt - # - uses: isort/isort-action@v1.1.1 - # with: - # sort-paths: antarest, tests - # requirementsFiles: "requirements-dev.txt" - # - name: Check with black - # uses: psf/black@stable - # with: - # # Version of Black should match the versions set in `requirements-dev.txt` - # version: "~=23.7.0" - # options: --check --diff - # - name: Check Typing (mypy) - # #continue-on-error: true - # run: | - # mypy --install-types --non-interactive + python-lint: + runs-on: ubuntu-20.04 + steps: + - name: Checkout github repo (+ download lfs dependencies) + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - uses: isort/isort-action@v1.1.1 + with: + sort-paths: antarest, tests + requirementsFiles: "requirements-dev.txt" + - name: Check with black + uses: psf/black@stable + with: + # Version of Black should match the versions set in `requirements-dev.txt` + version: "~=23.7.0" + options: --check --diff + - name: Check Typing (mypy) + #continue-on-error: true + run: | + mypy --install-types --non-interactive python-test: runs-on: ${{ matrix.os }} From 9729ea5bb1b9b7be15611e2368c55596c0a92324 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 15:09:26 +0100 Subject: [PATCH 13/37] wip --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f619ed434..099b6b9c12 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: strategy: max-parallel: 9 matrix: - os: [windows-latest] + os: [windows-latest, ubuntu-20.04] steps: - name: Checkout github repo (+ download lfs dependencies) From c574c15272f626115914d1b8f9cf0f47a028601a Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 15:14:17 +0100 Subject: [PATCH 14/37] wip --- .github/workflows/main.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 099b6b9c12..b2a5b5e6e6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -146,17 +146,17 @@ jobs: # NODE_OPTIONS: --max-old-space-size=8192 # DISABLE_ESLINT_PLUGIN: true - sonarcloud: - runs-on: ubuntu-20.04 - needs: [python-test, npm-test] - steps: - - uses: actions/checkout@v4 - - name: Download python coverage report - uses: actions/download-artifact@v4 - with: - name: python-code-coverage-report - - name: SonarCloud Scan - uses: sonarsource/sonarcloud-github-action@v3.1.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file + # sonarcloud: + # runs-on: ubuntu-20.04 + # needs: [python-test, npm-test] + # steps: + # - uses: actions/checkout@v4 + # - name: Download python coverage report + # uses: actions/download-artifact@v4 + # with: + # name: python-code-coverage-report + # - name: SonarCloud Scan + # uses: sonarsource/sonarcloud-github-action@v3.1.0 + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From 7cb32e612995363937acf4b4bd6c200150722c11 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 15:31:20 +0100 Subject: [PATCH 15/37] wip --- .github/workflows/main.yml | 2 +- tests/storage/conftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b2a5b5e6e6..9160b524a5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: strategy: max-parallel: 9 matrix: - os: [windows-latest, ubuntu-20.04] + os: [windows-latest] steps: - name: Checkout github repo (+ download lfs dependencies) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index dd19f75005..852828100e 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path, PureWindowsPath +from pathlib import Path, PurePosixPath from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -347,4 +347,4 @@ def __init__(self, config: Config): @pytest.fixture def tmp_path_posix(tmp_path: Path) -> Path: - return Path(PureWindowsPath(tmp_path).as_posix()) + return PurePosixPath(tmp_path) From 63a31322ada0e3c04851addd566f2e38c99e1498 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 15:42:59 +0100 Subject: [PATCH 16/37] wip debug --- tests/storage/business/test_raw_study_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 028a7c5b7f..f1de5d9a28 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -207,6 +207,7 @@ def test_assert_study_not_exist(tmp_path: str, project_path) -> None: @pytest.mark.unit_test def test_create(tmp_path_posix: Path, project_path: Path) -> None: + print("tmp_path_posix", tmp_path_posix) study = Mock() data = {"antares": {"caption": None}} study.get.return_value = data From 61d3d001c93a507e1c88836593267729b939a101 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 15:50:42 +0100 Subject: [PATCH 17/37] debug --- tests/storage/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 852828100e..385e44375f 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -347,4 +347,5 @@ def __init__(self, config: Config): @pytest.fixture def tmp_path_posix(tmp_path: Path) -> Path: - return PurePosixPath(tmp_path) + tmp_path_windows = PureWindowsPath(tmp_path) + return PurePosixPath(tmp_path_windows) From e9ee35d8b97ec665f7d58ddc4e115edd14e37b88 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 15:54:38 +0100 Subject: [PATCH 18/37] debug --- tests/storage/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 385e44375f..393f85471c 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -347,5 +347,8 @@ def __init__(self, config: Config): @pytest.fixture def tmp_path_posix(tmp_path: Path) -> Path: + print("in fixture tmp path", tmp_path) tmp_path_windows = PureWindowsPath(tmp_path) + print("in fixture tmp tmp_path_windows", tmp_path_windows) + print("in fixture tmp tmp path posix", PurePosixPath(tmp_path_windows)) return PurePosixPath(tmp_path_windows) From 1663c592716ca8ac4fd85655f14cc1814735b73e Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 16:03:14 +0100 Subject: [PATCH 19/37] debug --- tests/storage/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 393f85471c..c6880c4618 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -350,5 +350,5 @@ def tmp_path_posix(tmp_path: Path) -> Path: print("in fixture tmp path", tmp_path) tmp_path_windows = PureWindowsPath(tmp_path) print("in fixture tmp tmp_path_windows", tmp_path_windows) - print("in fixture tmp tmp path posix", PurePosixPath(tmp_path_windows)) - return PurePosixPath(tmp_path_windows) + print("in fixture tmp tmp path posix", PurePosixPath(tmp_path_windows.as_posix())) + return PurePosixPath(tmp_path_windows.as_posix()) From 78d474a57a3cd4fc41b7e5608206a1865ce73c2f Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 16:24:22 +0100 Subject: [PATCH 20/37] debug --- tests/storage/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index c6880c4618..9d312cb2c1 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path, PurePosixPath, PureWindowsPath +from pathlib import Path, PureWindowsPath from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -350,5 +350,6 @@ def tmp_path_posix(tmp_path: Path) -> Path: print("in fixture tmp path", tmp_path) tmp_path_windows = PureWindowsPath(tmp_path) print("in fixture tmp tmp_path_windows", tmp_path_windows) - print("in fixture tmp tmp path posix", PurePosixPath(tmp_path_windows.as_posix())) - return PurePosixPath(tmp_path_windows.as_posix()) + tmp_path_posix = Path(tmp_path_windows.as_posix()) + print("in fixture tmp tmp path posix", tmp_path_posix) + return tmp_path_posix From bc7ab117488f0a3f6ec97ebfc80f618cd7031850 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 16:37:13 +0100 Subject: [PATCH 21/37] debug --- tests/storage/conftest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 9d312cb2c1..a8ad7efefa 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path, PureWindowsPath +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -350,6 +350,10 @@ def tmp_path_posix(tmp_path: Path) -> Path: print("in fixture tmp path", tmp_path) tmp_path_windows = PureWindowsPath(tmp_path) print("in fixture tmp tmp_path_windows", tmp_path_windows) - tmp_path_posix = Path(tmp_path_windows.as_posix()) + tmp_path_posix = PurePosixPath(tmp_path_windows.as_posix()) print("in fixture tmp tmp path posix", tmp_path_posix) - return tmp_path_posix + + tmp_path_normal = Path(tmp_path_posix) + print("in fixture tmp tmp path posix", tmp_path_normal) + + return tmp_path_normal From 9a98403282aa4c1030add1ff27649bdcf98cd751 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 16:40:30 +0100 Subject: [PATCH 22/37] debug --- tests/storage/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index a8ad7efefa..dab27e1020 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path, PurePosixPath, PureWindowsPath +from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -353,7 +353,7 @@ def tmp_path_posix(tmp_path: Path) -> Path: tmp_path_posix = PurePosixPath(tmp_path_windows.as_posix()) print("in fixture tmp tmp path posix", tmp_path_posix) - tmp_path_normal = Path(tmp_path_posix) + tmp_path_normal = PurePath(tmp_path_posix) print("in fixture tmp tmp path posix", tmp_path_normal) return tmp_path_normal From 09ebc5bbd6ec5e01bafab9ff408bf0ca1c294941 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 16:44:04 +0100 Subject: [PATCH 23/37] debug --- tests/storage/conftest.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index dab27e1020..4cbdc6eaaf 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -352,8 +352,4 @@ def tmp_path_posix(tmp_path: Path) -> Path: print("in fixture tmp tmp_path_windows", tmp_path_windows) tmp_path_posix = PurePosixPath(tmp_path_windows.as_posix()) print("in fixture tmp tmp path posix", tmp_path_posix) - - tmp_path_normal = PurePath(tmp_path_posix) - print("in fixture tmp tmp path posix", tmp_path_normal) - - return tmp_path_normal + return tmp_path_posix From 132e11a16a3bf2e3ec7d531c573c59d8113e5ff8 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 16:49:29 +0100 Subject: [PATCH 24/37] debug --- tests/storage/business/test_raw_study_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index f1de5d9a28..30fe11bcd8 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -237,9 +237,10 @@ def test_create(tmp_path_posix: Path, project_path: Path) -> None: print("tmp_path_posix", tmp_path_posix) assert md.path == str(tmp_path_posix / "study1") path_study = tmp_path_posix / md.id - assert path_study.exists() + path_stud_os_specifix = Path(path_study) + assert path_stud_os_specifix.exists() - path_study_antares_infos = path_study / "study.antares" + path_study_antares_infos = path_stud_os_specifix / "study.antares" assert path_study_antares_infos.is_file() From d0e16c2a6cc282ccd2790b7574e1c59a808d62d6 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 17:58:02 +0100 Subject: [PATCH 25/37] debug --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9160b524a5..9840de7bc2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,7 +53,7 @@ jobs: pip install -r requirements-dev.txt - name: Test with pytest run: | - pytest --cov antarest --cov-report xml -n auto --dist=worksteal tests/storage/business/test_raw_study_service.py + pytest --cov antarest --cov-report xml -n auto --dist=worksteal tests/storage/business/test_import.py - name: Archive code coverage results if: matrix.os == 'ubuntu-20.04' uses: actions/upload-artifact@v4 From 31dc2b9e7cc6a36207dedca5221e22be35a6df8a Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 18:05:14 +0100 Subject: [PATCH 26/37] debug --- tests/storage/business/test_import.py | 3 ++- tests/storage/business/test_raw_study_service.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/storage/business/test_import.py b/tests/storage/business/test_import.py index 0958b28fde..ed11cd80e0 100644 --- a/tests/storage/business/test_import.py +++ b/tests/storage/business/test_import.py @@ -29,7 +29,8 @@ @pytest.mark.unit_test def test_import_study(tmp_path_posix: Path) -> None: name = "my-study" - study_path = tmp_path_posix / name + study_path_posix = tmp_path_posix / name + study_path = Path(study_path_posix) study_path.mkdir() (study_path / "study.antares").touch() diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 30fe11bcd8..c3f6fd2064 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -207,7 +207,6 @@ def test_assert_study_not_exist(tmp_path: str, project_path) -> None: @pytest.mark.unit_test def test_create(tmp_path_posix: Path, project_path: Path) -> None: - print("tmp_path_posix", tmp_path_posix) study = Mock() data = {"antares": {"caption": None}} study.get.return_value = data @@ -234,7 +233,6 @@ def test_create(tmp_path_posix: Path, project_path: Path) -> None: ) md = study_service.create(metadata) - print("tmp_path_posix", tmp_path_posix) assert md.path == str(tmp_path_posix / "study1") path_study = tmp_path_posix / md.id path_stud_os_specifix = Path(path_study) From d1e2376a7ef78a564a7bf68c364d6eeb6da27744 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 18:33:20 +0100 Subject: [PATCH 27/37] debug --- antarest/study/model.py | 4 ---- tests/storage/business/test_raw_study_service.py | 8 ++++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index 968800d31c..4dca7463e8 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -289,10 +289,6 @@ def __eq__(self, other: t.Any) -> bool: def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} - @validates("path") - def validate_email(self, key, path) -> str: - return normalize_path(path) - @validates("folder") def validate_folder(self, key, folder) -> str: return normalize_path(folder) diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index c3f6fd2064..6669fbd182 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -206,14 +206,14 @@ def test_assert_study_not_exist(tmp_path: str, project_path) -> None: @pytest.mark.unit_test -def test_create(tmp_path_posix: Path, project_path: Path) -> None: +def test_create(tmp_path: Path, project_path: Path) -> None: study = Mock() data = {"antares": {"caption": None}} study.get.return_value = data study_factory = Mock() study_factory.create_from_fs.return_value = FileStudy(Mock(), study) - config = build_config(tmp_path_posix) + config = build_config(tmp_path) study_service = RawStudyService( config=config, cache=Mock(), @@ -233,8 +233,8 @@ def test_create(tmp_path_posix: Path, project_path: Path) -> None: ) md = study_service.create(metadata) - assert md.path == str(tmp_path_posix / "study1") - path_study = tmp_path_posix / md.id + assert md.path == str(tmp_path / "study1") + path_study = tmp_path / md.id path_stud_os_specifix = Path(path_study) assert path_stud_os_specifix.exists() From 058f898f412bae44b7422ada5736e82178ebf437 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Tue, 28 Jan 2025 18:45:02 +0100 Subject: [PATCH 28/37] debug --- tests/storage/business/test_import.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/storage/business/test_import.py b/tests/storage/business/test_import.py index ed11cd80e0..7b8af36f65 100644 --- a/tests/storage/business/test_import.py +++ b/tests/storage/business/test_import.py @@ -27,10 +27,9 @@ @pytest.mark.unit_test -def test_import_study(tmp_path_posix: Path) -> None: +def test_import_study(tmp_path: Path) -> None: name = "my-study" - study_path_posix = tmp_path_posix / name - study_path = Path(study_path_posix) + study_path = tmp_path / name study_path.mkdir() (study_path / "study.antares").touch() @@ -53,7 +52,7 @@ def test_import_study(tmp_path_posix: Path) -> None: study_service.get.return_value = data # first test importing a study for an archived study with `.zip` format - study_service.get_study_path.return_value = tmp_path_posix / "other-study-zip" + study_service.get_study_path.return_value = tmp_path / "other-study-zip" filepath_zip = shutil.make_archive(str(study_path.absolute()), "zip", study_path) shutil.rmtree(study_path) @@ -63,20 +62,20 @@ def test_import_study(tmp_path_posix: Path) -> None: md = RawStudy( id="other-study-zip", workspace=DEFAULT_WORKSPACE_NAME, - path=tmp_path_posix / "other-study-zip", + path=tmp_path / "other-study-zip", additional_data=StudyAdditionalData(), groups=["fake_group_1", "fake_group_2"], ) with path_zip.open("rb") as input_file: md = study_service.import_study(md, input_file) - assert md.path == f"{tmp_path_posix}/other-study-zip" + assert md.path == f"{tmp_path}{os.sep}other-study-zip" # assert that importing file into a created study does not alter its group assert md.groups == ["fake_group_1", "fake_group_2"] - shutil.rmtree(tmp_path_posix / "other-study-zip") + shutil.rmtree(tmp_path / "other-study-zip") # second test for an archived study with a `.7z` format - study_service.get_study_path.return_value = tmp_path_posix / "other-study-7zip" + study_service.get_study_path.return_value = tmp_path / "other-study-7zip" study_path.mkdir() (study_path / "study.antares").touch() @@ -90,17 +89,17 @@ def test_import_study(tmp_path_posix: Path) -> None: md = RawStudy( id="other-study-7zip", workspace=DEFAULT_WORKSPACE_NAME, - path=tmp_path_posix / "other-study-7zip", + path=tmp_path / "other-study-7zip", additional_data=StudyAdditionalData(), groups=["fake_group_1", "fake_group_2"], ) with filepath_7zip.open("rb") as input_file: md = study_service.import_study(md, input_file) - assert md.path == f"{tmp_path_posix}/other-study-7zip" + assert md.path == f"{tmp_path}{os.sep}other-study-7zip" # assert that importing file into a created study does not alter its group assert md.groups == ["fake_group_1", "fake_group_2"] - shutil.rmtree(tmp_path_posix / "other-study-7zip") + shutil.rmtree(tmp_path / "other-study-7zip") # test for an unsupported archive format with pytest.raises(BadArchiveContent, match="Unsupported archive format"): From e06ed1cbbe13686b564c167b23a8ccdd97ef3280 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Wed, 29 Jan 2025 10:08:01 +0100 Subject: [PATCH 29/37] revert tests --- .github/workflows/main.yml | 2 +- tests/storage/business/test_raw_study_service.py | 2 +- tests/storage/conftest.py | 12 +----------- tests/storage/test_service.py | 9 +++++---- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9840de7bc2..b6eaff423f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,7 +53,7 @@ jobs: pip install -r requirements-dev.txt - name: Test with pytest run: | - pytest --cov antarest --cov-report xml -n auto --dist=worksteal tests/storage/business/test_import.py + pytest --cov antarest --cov-report xml -n auto --dist=worksteal - name: Archive code coverage results if: matrix.os == 'ubuntu-20.04' uses: actions/upload-artifact@v4 diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 6669fbd182..6b14b05c77 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -455,7 +455,7 @@ def test_copy_study( md = study_service.copy(src_md, "dst_name", groups) md_id = md.id tmp_path_as_posix = tmp_path.as_posix() - assert str(md.path) == f"{tmp_path_as_posix}/{md_id}" + assert str(md.path) == f"{tmp_path_as_posix}{os.sep}{md_id}" assert md.public_mode == PublicMode.NONE assert md.groups == groups study.get.assert_called_once_with(["study"]) diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 4cbdc6eaaf..297a367d08 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -12,7 +12,7 @@ import datetime import uuid -from pathlib import Path, PurePosixPath, PureWindowsPath +from pathlib import Path from typing import Callable, Dict, List, Optional, Union from unittest.mock import Mock @@ -343,13 +343,3 @@ def get_all(self, owner: Optional[int] = None) -> List[FileDownload]: class SimpleFileTransferManager(FileTransferManager): def __init__(self, config: Config): super().__init__(FileDownloadRepositoryMock(), Mock(), config) - - -@pytest.fixture -def tmp_path_posix(tmp_path: Path) -> Path: - print("in fixture tmp path", tmp_path) - tmp_path_windows = PureWindowsPath(tmp_path) - print("in fixture tmp tmp_path_windows", tmp_path_windows) - tmp_path_posix = PurePosixPath(tmp_path_windows.as_posix()) - print("in fixture tmp tmp path posix", tmp_path_posix) - return tmp_path_posix diff --git a/tests/storage/test_service.py b/tests/storage/test_service.py index 637fa02842..f8b1770bd1 100644 --- a/tests/storage/test_service.py +++ b/tests/storage/test_service.py @@ -11,6 +11,7 @@ # This file is part of the Antares project. import contextlib +import os import textwrap import typing as t import uuid @@ -318,7 +319,7 @@ def test_partial_sync_studies_from_disk() -> None: mb = RawStudy(id="b", path="b") mc = RawStudy( id="c", - path="directory/c", + path=f"directory{os.sep}c", name="c", content_status=StudyContentStatus.WARNING, workspace=DEFAULT_WORKSPACE_NAME, @@ -326,12 +327,12 @@ def test_partial_sync_studies_from_disk() -> None: ) md = RawStudy( id="d", - path="directory/d", + path=f"directory{os.sep}d", missing=datetime.utcnow() - timedelta(MAX_MISSING_STUDY_TIMEOUT + 1), ) me = RawStudy( id="e", - path="directory/e", + path=f"directory{os.sep}e", created_at=now, missing=datetime.utcnow() - timedelta(MAX_MISSING_STUDY_TIMEOUT - 1), ) @@ -350,7 +351,7 @@ def test_partial_sync_studies_from_disk() -> None: repository.save.assert_called_with( RawStudy( id=ANY, - path="directory/f", + path=f"directory{os.sep}f", name="f", folder="directory/f", created_at=ANY, From aa633a808ac3a0955da5d38d48f53b74c3bd4b8d Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Wed, 29 Jan 2025 15:31:17 +0100 Subject: [PATCH 30/37] wip --- antarest/study/service.py | 2 +- tests/storage/business/test_raw_study_service.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/antarest/study/service.py b/antarest/study/service.py index 9b7d259e94..31bb80fa1f 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -993,7 +993,7 @@ def sync_studies_on_disk( study_paths = [study.path for study in all_studies if study.missing is None] missing_studies = {study.path: study for study in all_studies if study.missing is not None} for folder in folders: - study_path = str(folder.path.as_posix()) + study_path = str(folder.path) if study_path not in study_paths: try: if study_path not in missing_studies.keys(): diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 6b14b05c77..60a6bce5f0 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -454,8 +454,7 @@ def test_copy_study( ) md = study_service.copy(src_md, "dst_name", groups) md_id = md.id - tmp_path_as_posix = tmp_path.as_posix() - assert str(md.path) == f"{tmp_path_as_posix}{os.sep}{md_id}" + assert str(md.path) == f"{tmp_path}{os.sep}{md_id}" assert md.public_mode == PublicMode.NONE assert md.groups == groups study.get.assert_called_once_with(["study"]) From 3cb28c5247334fef37169efdfc4a82bbcd17f386 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Wed, 29 Jan 2025 15:57:02 +0100 Subject: [PATCH 31/37] wip --- .github/workflows/main.yml | 192 +++++++++++++++++----------------- antarest/study/model.py | 7 +- tests/storage/test_service.py | 2 +- 3 files changed, 102 insertions(+), 99 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b6eaff423f..a5207a3d20 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: strategy: max-parallel: 9 matrix: - os: [windows-latest] + os: [windows-latest, ubuntu-20.04] steps: - name: Checkout github repo (+ download lfs dependencies) @@ -61,102 +61,102 @@ jobs: name: python-code-coverage-report path: coverage.xml - # npm-setup: - # runs-on: ubuntu-20.04 - # steps: - # - name: Checkout github repo - # uses: actions/checkout@v4 - # - name: Set up Node.js - # uses: actions/setup-node@v4 - # with: - # node-version: 22.13.0 - # - name: Cache node modules - # uses: actions/cache@v4 - # with: - # path: webapp/node_modules - # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - # restore-keys: | - # ${{ runner.os }}-webapp-node-modules- - # ${{ runner.os }}- - # save-always: true - # - name: Install dependencies - # run: npm install - # working-directory: webapp + npm-setup: + runs-on: ubuntu-20.04 + steps: + - name: Checkout github repo + uses: actions/checkout@v4 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.13.0 + - name: Cache node modules + uses: actions/cache@v4 + with: + path: webapp/node_modules + key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-webapp-node-modules- + ${{ runner.os }}- + save-always: true + - name: Install dependencies + run: npm install + working-directory: webapp - # npm-lint: - # needs: npm-setup - # runs-on: ubuntu-20.04 - # steps: - # - name: Checkout github repo - # uses: actions/checkout@v4 - # - name: Set up Node.js - # uses: actions/setup-node@v4 - # with: - # node-version: 22.13.0 - # - name: Restore node modules - # uses: actions/cache@v4 - # with: - # path: webapp/node_modules - # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - # save-always: true - # - name: Lint - # run: npm run lint - # working-directory: webapp + npm-lint: + needs: npm-setup + runs-on: ubuntu-20.04 + steps: + - name: Checkout github repo + uses: actions/checkout@v4 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.13.0 + - name: Restore node modules + uses: actions/cache@v4 + with: + path: webapp/node_modules + key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + save-always: true + - name: Lint + run: npm run lint + working-directory: webapp - # npm-test: - # needs: npm-setup - # runs-on: ubuntu-20.04 - # steps: - # - name: Checkout github repo - # uses: actions/checkout@v4 - # - name: Set up Node.js - # uses: actions/setup-node@v4 - # with: - # node-version: 22.13.0 - # - name: Restore node modules - # uses: actions/cache@v4 - # with: - # path: webapp/node_modules - # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - # save-always: true - # - name: Test - # run: npm run test - # working-directory: webapp + npm-test: + needs: npm-setup + runs-on: ubuntu-20.04 + steps: + - name: Checkout github repo + uses: actions/checkout@v4 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.13.0 + - name: Restore node modules + uses: actions/cache@v4 + with: + path: webapp/node_modules + key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + save-always: true + - name: Test + run: npm run test + working-directory: webapp - # npm-build: - # needs: npm-setup - # runs-on: ubuntu-20.04 - # steps: - # - name: Checkout github repo - # uses: actions/checkout@v4 - # - name: Set up Node.js - # uses: actions/setup-node@v4 - # with: - # node-version: 22.13.0 - # - name: Restore node modules - # uses: actions/cache@v4 - # with: - # path: webapp/node_modules - # key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} - # save-always: true - # - name: Build - # run: npm run build - # working-directory: webapp - # env: - # NODE_OPTIONS: --max-old-space-size=8192 - # DISABLE_ESLINT_PLUGIN: true + npm-build: + needs: npm-setup + runs-on: ubuntu-20.04 + steps: + - name: Checkout github repo + uses: actions/checkout@v4 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.13.0 + - name: Restore node modules + uses: actions/cache@v4 + with: + path: webapp/node_modules + key: ${{ runner.os }}-webapp-node-modules-${{ hashFiles('webapp/package-lock.json') }} + save-always: true + - name: Build + run: npm run build + working-directory: webapp + env: + NODE_OPTIONS: --max-old-space-size=8192 + DISABLE_ESLINT_PLUGIN: true - # sonarcloud: - # runs-on: ubuntu-20.04 - # needs: [python-test, npm-test] - # steps: - # - uses: actions/checkout@v4 - # - name: Download python coverage report - # uses: actions/download-artifact@v4 - # with: - # name: python-code-coverage-report - # - name: SonarCloud Scan - # uses: sonarsource/sonarcloud-github-action@v3.1.0 - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file + sonarcloud: + runs-on: ubuntu-20.04 + needs: [python-test, npm-test] + steps: + - uses: actions/checkout@v4 + - name: Download python coverage report + uses: actions/download-artifact@v4 + with: + name: python-code-coverage-report + - name: SonarCloud Scan + uses: sonarsource/sonarcloud-github-action@v3.1.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/antarest/study/model.py b/antarest/study/model.py index 4dca7463e8..fb65e6e47c 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -231,7 +231,7 @@ class Study(Base): # type: ignore created_at = Column(DateTime, index=True) updated_at = Column(DateTime, index=True) last_access = Column(DateTime) - path = Column(String) + path = Column(String()) folder = Column(String, nullable=True, index=True) parent_id = Column(String(36), ForeignKey("study.id", name="fk_study_study_id"), index=True) public_mode = Column(Enum(PublicMode), default=PublicMode.NONE) @@ -291,12 +291,15 @@ def to_json_summary(self) -> t.Any: @validates("folder") def validate_folder(self, key, folder) -> str: + """ + We want to store the path in posix format in the database, even on windows. + """ return normalize_path(folder) def normalize_path(path: str) -> str: """ - We want to store the path in posix format in the database, even on windows. + Turns any path including a windows path (with \ separator) to a posix path (with / separator). """ if not path: return path diff --git a/tests/storage/test_service.py b/tests/storage/test_service.py index f8b1770bd1..a6d36131f9 100644 --- a/tests/storage/test_service.py +++ b/tests/storage/test_service.py @@ -387,7 +387,7 @@ def test_windows_path_scan() -> None: repository.save.assert_called_with( RawStudy( id=ANY, - path="directory/b", + path=f"directory{os.sep}b", name="b", folder="directory/b", created_at=ANY, From b7a853c3ebfb7d35785efd983b45a369016a14a5 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Wed, 29 Jan 2025 16:43:50 +0100 Subject: [PATCH 32/37] debug --- tests/storage/test_service.py | 38 +---------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/tests/storage/test_service.py b/tests/storage/test_service.py index a6d36131f9..06a7d45b0f 100644 --- a/tests/storage/test_service.py +++ b/tests/storage/test_service.py @@ -17,7 +17,7 @@ import uuid from configparser import MissingSectionHeaderError from datetime import datetime, timedelta, timezone -from pathlib import Path, PureWindowsPath +from pathlib import Path from unittest.mock import ANY, Mock, call, patch, seal import pytest @@ -362,42 +362,6 @@ def test_partial_sync_studies_from_disk() -> None: ) -# noinspection PyArgumentList -@pytest.mark.unit_test -def test_windows_path_scan() -> None: - """ - In this test scenarion only study a exist in database, we call the sync with a folder containing study b. - - The input folder is a windows folder. - - We expect the sync to save the study b with a posix path and not a windows path. - - Note : study a won't get deleted, the scan is partial as we give a directory argument. - """ - ma = RawStudy(id="a", path="a") - fb = StudyFolder(path=PureWindowsPath("directory\\b"), workspace=DEFAULT_WORKSPACE_NAME, groups=[]) - - repository = Mock() - repository.get_all_raw.side_effect = [[ma]] - config = Config(storage=StorageConfig(workspaces={DEFAULT_WORKSPACE_NAME: WorkspaceConfig()})) - service = build_study_service(Mock(), repository, config) - - service.sync_studies_on_disk([fb], directory=PureWindowsPath("directory")) - - repository.save.assert_called_with( - RawStudy( - id=ANY, - path=f"directory{os.sep}b", - name="b", - folder="directory/b", - created_at=ANY, - missing=None, - public_mode=PublicMode.FULL, - workspace=DEFAULT_WORKSPACE_NAME, - ) - ) - - @with_db_context def test_remove_duplicate(db_session: Session) -> None: with db_session: From 916969f41c0d0df827b4104ad8d09a65d4650443 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Wed, 29 Jan 2025 16:56:08 +0100 Subject: [PATCH 33/37] debug --- tests/storage/business/test_raw_study_service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 60a6bce5f0..54ff697e92 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -235,10 +235,9 @@ def test_create(tmp_path: Path, project_path: Path) -> None: assert md.path == str(tmp_path / "study1") path_study = tmp_path / md.id - path_stud_os_specifix = Path(path_study) - assert path_stud_os_specifix.exists() + assert path_study.exists() - path_study_antares_infos = path_stud_os_specifix / "study.antares" + path_study_antares_infos = path_study / "study.antares" assert path_study_antares_infos.is_file() From a8d1f255e89467733fc6de30981b76a6f1acd6f3 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Thu, 30 Jan 2025 09:45:17 +0100 Subject: [PATCH 34/37] fix build --- antarest/study/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index fb65e6e47c..1be13fbdc1 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -289,8 +289,8 @@ def __eq__(self, other: t.Any) -> bool: def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} - @validates("folder") - def validate_folder(self, key, folder) -> str: + @validates("folder") # type: ignore + def validate_folder(self, key: str, folder: str) -> str: """ We want to store the path in posix format in the database, even on windows. """ From bd5538ad8e28abad28ceb68868bfc4d5b2aed19a Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Thu, 30 Jan 2025 09:48:02 +0100 Subject: [PATCH 35/37] refactor typing --- antarest/study/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index 1be13fbdc1..7d1b835af1 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -290,14 +290,14 @@ def to_json_summary(self) -> t.Any: return {"id": self.id, "name": self.name} @validates("folder") # type: ignore - def validate_folder(self, key: str, folder: str) -> str: + def validate_folder(self, key: str, folder: t.Optional[str]) -> t.Optional[str]: """ We want to store the path in posix format in the database, even on windows. """ return normalize_path(folder) -def normalize_path(path: str) -> str: +def normalize_path(path: t.Optional[str]) -> str: """ Turns any path including a windows path (with \ separator) to a posix path (with / separator). """ From e3b642d3137cc99d213bfbbd19229ca811b090a2 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Thu, 30 Jan 2025 09:48:41 +0100 Subject: [PATCH 36/37] refactor typing --- antarest/study/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index 7d1b835af1..f3d26e4c4b 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -297,7 +297,7 @@ def validate_folder(self, key: str, folder: t.Optional[str]) -> t.Optional[str]: return normalize_path(folder) -def normalize_path(path: t.Optional[str]) -> str: +def normalize_path(path: t.Optional[str]) -> t.Optional[str]: """ Turns any path including a windows path (with \ separator) to a posix path (with / separator). """ From e987d246442270665b6988aaa668699a3828dea8 Mon Sep 17 00:00:00 2001 From: Anis SMAIL Date: Thu, 30 Jan 2025 10:04:29 +0100 Subject: [PATCH 37/37] remove uselless pure windows path --- antarest/study/model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index f3d26e4c4b..004c836844 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -16,7 +16,7 @@ import typing as t import uuid from datetime import datetime, timedelta -from pathlib import Path, PureWindowsPath +from pathlib import Path, PurePath from antares.study.version import StudyVersion from pydantic import BeforeValidator, ConfigDict, Field, PlainSerializer, computed_field, field_validator @@ -303,8 +303,8 @@ def normalize_path(path: t.Optional[str]) -> t.Optional[str]: """ if not path: return path - windows_path = PureWindowsPath(path) - return windows_path.as_posix() + pure_path = PurePath(path) + return pure_path.as_posix() class RawStudy(Study):