From ecbcac1314dbf42f290930abc2423e3f3d820c65 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 10:00:17 -0700 Subject: [PATCH 1/9] cp to tests --- tests/schema_privileges.py | 35 ++++++++++++ tests/test_privileges.py | 109 +++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 tests/schema_privileges.py create mode 100644 tests/test_privileges.py diff --git a/tests/schema_privileges.py b/tests/schema_privileges.py new file mode 100644 index 000000000..8b39e4aa1 --- /dev/null +++ b/tests/schema_privileges.py @@ -0,0 +1,35 @@ +import datajoint as dj + +schema = dj.Schema() + + +@schema +class Parent(dj.Lookup): + definition = """ + id: int + """ + contents = [(1,)] + + +@schema +class Child(dj.Computed): + definition = """ + -> Parent + """ + + def make(self, key): + self.insert1(key) + + +@schema +class NoAccess(dj.Lookup): + definition = """ + string: varchar(10) + """ + + +@schema +class NoAccessAgain(dj.Manual): + definition = """ + -> NoAccess + """ diff --git a/tests/test_privileges.py b/tests/test_privileges.py new file mode 100644 index 000000000..f32a1103f --- /dev/null +++ b/tests/test_privileges.py @@ -0,0 +1,109 @@ +import importlib +import datajoint as dj +from . import schema, CONN_INFO_ROOT, PREFIX +from . import schema_privileges as pipeline +from nose.tools import assert_true, raises + +namespace = locals() + + +class TestUnprivileged: + @classmethod + def setup_class(cls): + """A connection with only SELECT privilege to djtest schemas""" + cls.connection = dj.conn( + host=CONN_INFO_ROOT["host"], user="djview", password="djview", reset=True + ) + + @raises(dj.DataJointError) + def test_fail_create_schema(self): + """creating a schema with no CREATE privilege""" + return dj.Schema("forbidden_schema", namespace, connection=self.connection) + + @raises(dj.DataJointError) + def test_insert_failure(self): + unprivileged = dj.Schema( + schema.schema.database, namespace, connection=self.connection + ) + unprivileged.spawn_missing_classes() + assert_true( + issubclass(Language, dj.Lookup) + and len(Language()) == len(schema.Language()), + "failed to spawn missing classes", + ) + Language().insert1(("Socrates", "Greek")) + + @raises(dj.DataJointError) + def test_failure_to_create_table(self): + unprivileged = dj.Schema( + schema.schema.database, namespace, connection=self.connection + ) + + @unprivileged + class Try(dj.Manual): + definition = """ # should not matter really + id : int + --- + value : float + """ + + Try().insert1((1, 1.5)) + + +class TestSubset: + USER = "djsubset" + + @classmethod + def setup_class(cls): + conn = dj.conn( + host=CONN_INFO_ROOT["host"], + user=CONN_INFO_ROOT["user"], + password=CONN_INFO_ROOT["password"], + reset=True, + ) + pipeline.schema.activate(f"{PREFIX}_pipeline") + conn.query( + f""" + CREATE USER IF NOT EXISTS '{cls.USER}'@'%%' + IDENTIFIED BY '{cls.USER}' + """ + ) + conn.query( + f""" + GRANT SELECT, INSERT, UPDATE, DELETE + ON `{PREFIX}_pipeline`.`#parent` + TO '{cls.USER}'@'%%' + """ + ) + conn.query( + f""" + GRANT SELECT, INSERT, UPDATE, DELETE + ON `{PREFIX}_pipeline`.`__child` + TO '{cls.USER}'@'%%' + """ + ) + cls.connection = dj.conn( + host=CONN_INFO_ROOT["host"], + user=cls.USER, + password=cls.USER, + reset=True, + ) + + @classmethod + def teardown_class(cls): + conn = dj.conn( + host=CONN_INFO_ROOT["host"], + user=CONN_INFO_ROOT["user"], + password=CONN_INFO_ROOT["password"], + reset=True, + ) + conn.query(f"DROP USER {cls.USER}") + conn.query(f"DROP DATABASE {PREFIX}_pipeline") + + def test_populate_activate(self): + importlib.reload(pipeline) + pipeline.schema.activate( + f"{PREFIX}_pipeline", create_schema=True, create_tables=False + ) + pipeline.Child.populate() + assert pipeline.Child.progress(display=False)[0] == 0 From fdd435288e4268689e8f466575cc2cc612a36c8c Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 10:14:58 -0700 Subject: [PATCH 2/9] Migrate TestUnprivileged --- tests/schema_privileges.py | 11 +++--- tests/test_privileges.py | 72 ++++++++++++++++++++------------------ 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/tests/schema_privileges.py b/tests/schema_privileges.py index 8b39e4aa1..57f8ebb74 100644 --- a/tests/schema_privileges.py +++ b/tests/schema_privileges.py @@ -1,9 +1,7 @@ import datajoint as dj +import inspect -schema = dj.Schema() - -@schema class Parent(dj.Lookup): definition = """ id: int @@ -11,7 +9,6 @@ class Parent(dj.Lookup): contents = [(1,)] -@schema class Child(dj.Computed): definition = """ -> Parent @@ -21,15 +18,17 @@ def make(self, key): self.insert1(key) -@schema class NoAccess(dj.Lookup): definition = """ string: varchar(10) """ -@schema class NoAccessAgain(dj.Manual): definition = """ -> NoAccess """ + + +LOCALS_PRIV = {k: v for k, v in locals().items() if inspect.isclass(v)} +__all__ = list(LOCALS_PRIV) \ No newline at end of file diff --git a/tests/test_privileges.py b/tests/test_privileges.py index f32a1103f..35ca35958 100644 --- a/tests/test_privileges.py +++ b/tests/test_privileges.py @@ -1,42 +1,45 @@ +import os +import pytest import importlib import datajoint as dj from . import schema, CONN_INFO_ROOT, PREFIX -from . import schema_privileges as pipeline -from nose.tools import assert_true, raises +from . import schema_privileges namespace = locals() +@pytest.fixture +def connection_djview(connection_root): + """ + A connection with only SELECT privilege to djtest schemas. + Requires connection_root fixture so that `djview` user exists. + """ + connection = dj.conn( + host=os.getenv("DJ_HOST"), + user="djview", + password="djview", + reset=True, + ) + yield connection -class TestUnprivileged: - @classmethod - def setup_class(cls): - """A connection with only SELECT privilege to djtest schemas""" - cls.connection = dj.conn( - host=CONN_INFO_ROOT["host"], user="djview", password="djview", reset=True - ) - @raises(dj.DataJointError) - def test_fail_create_schema(self): +class TestUnprivileged: + def test_fail_create_schema(self, connection_djview): """creating a schema with no CREATE privilege""" - return dj.Schema("forbidden_schema", namespace, connection=self.connection) + with pytest.raises(dj.DataJointError): + return dj.Schema("forbidden_schema", namespace, connection=connection_djview) - @raises(dj.DataJointError) - def test_insert_failure(self): + def test_insert_failure(self, connection_djview, schema_any): unprivileged = dj.Schema( - schema.schema.database, namespace, connection=self.connection + schema_any.database, namespace, connection=connection_djview ) unprivileged.spawn_missing_classes() - assert_true( - issubclass(Language, dj.Lookup) - and len(Language()) == len(schema.Language()), - "failed to spawn missing classes", - ) - Language().insert1(("Socrates", "Greek")) + assert issubclass(Language, dj.Lookup) and len(Language()) == len(schema.Language()), "failed to spawn missing classes" + with pytest.raises(dj.DataJointError): + Language().insert1(("Socrates", "Greek")) - @raises(dj.DataJointError) - def test_failure_to_create_table(self): + def test_failure_to_create_table(self, connection_djview, schema_any): unprivileged = dj.Schema( - schema.schema.database, namespace, connection=self.connection + schema_any.database, namespace, connection=connection_djview ) @unprivileged @@ -47,7 +50,8 @@ class Try(dj.Manual): value : float """ - Try().insert1((1, 1.5)) + with pytest.raises(dj.DataJointError): + Try().insert1((1, 1.5)) class TestSubset: @@ -61,7 +65,7 @@ def setup_class(cls): password=CONN_INFO_ROOT["password"], reset=True, ) - pipeline.schema.activate(f"{PREFIX}_pipeline") + schema_privileges.schema.activate(f"{PREFIX}_schema_privileges") conn.query( f""" CREATE USER IF NOT EXISTS '{cls.USER}'@'%%' @@ -71,14 +75,14 @@ def setup_class(cls): conn.query( f""" GRANT SELECT, INSERT, UPDATE, DELETE - ON `{PREFIX}_pipeline`.`#parent` + ON `{PREFIX}_schema_privileges`.`#parent` TO '{cls.USER}'@'%%' """ ) conn.query( f""" GRANT SELECT, INSERT, UPDATE, DELETE - ON `{PREFIX}_pipeline`.`__child` + ON `{PREFIX}_schema_privileges`.`__child` TO '{cls.USER}'@'%%' """ ) @@ -98,12 +102,12 @@ def teardown_class(cls): reset=True, ) conn.query(f"DROP USER {cls.USER}") - conn.query(f"DROP DATABASE {PREFIX}_pipeline") + conn.query(f"DROP DATABASE {PREFIX}_schema_privileges") def test_populate_activate(self): - importlib.reload(pipeline) - pipeline.schema.activate( - f"{PREFIX}_pipeline", create_schema=True, create_tables=False + importlib.reload(schema_privileges) + schema_privileges.schema.activate( + f"{PREFIX}_schema_privileges", create_schema=True, create_tables=False ) - pipeline.Child.populate() - assert pipeline.Child.progress(display=False)[0] == 0 + schema_privileges.Child.populate() + assert schema_privileges.Child.progress(display=False)[0] == 0 From 37248ff60d633085b4534c98d0f81e82bd508b31 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 10:15:02 -0700 Subject: [PATCH 3/9] Format with black --- tests/schema_privileges.py | 2 +- tests/test_privileges.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/schema_privileges.py b/tests/schema_privileges.py index 57f8ebb74..b53d6b264 100644 --- a/tests/schema_privileges.py +++ b/tests/schema_privileges.py @@ -31,4 +31,4 @@ class NoAccessAgain(dj.Manual): LOCALS_PRIV = {k: v for k, v in locals().items() if inspect.isclass(v)} -__all__ = list(LOCALS_PRIV) \ No newline at end of file +__all__ = list(LOCALS_PRIV) diff --git a/tests/test_privileges.py b/tests/test_privileges.py index 35ca35958..0cb807a6b 100644 --- a/tests/test_privileges.py +++ b/tests/test_privileges.py @@ -7,6 +7,7 @@ namespace = locals() + @pytest.fixture def connection_djview(connection_root): """ @@ -26,14 +27,18 @@ class TestUnprivileged: def test_fail_create_schema(self, connection_djview): """creating a schema with no CREATE privilege""" with pytest.raises(dj.DataJointError): - return dj.Schema("forbidden_schema", namespace, connection=connection_djview) + return dj.Schema( + "forbidden_schema", namespace, connection=connection_djview + ) def test_insert_failure(self, connection_djview, schema_any): unprivileged = dj.Schema( schema_any.database, namespace, connection=connection_djview ) unprivileged.spawn_missing_classes() - assert issubclass(Language, dj.Lookup) and len(Language()) == len(schema.Language()), "failed to spawn missing classes" + assert issubclass(Language, dj.Lookup) and len(Language()) == len( + schema.Language() + ), "failed to spawn missing classes" with pytest.raises(dj.DataJointError): Language().insert1(("Socrates", "Greek")) From 08fcac056c1d3207db160d908ba3dc3023805afe Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 10:30:56 -0700 Subject: [PATCH 4/9] Separate fixture for DB creds dict --- tests/conftest.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9d697ef47..470c2f440 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ import datajoint as dj from packaging import version +from typing import Dict import os from os import environ, remove import minio @@ -52,12 +53,17 @@ def enable_filepath_feature(monkeypatch): @pytest.fixture(scope="session") -def connection_root_bare(): - connection = dj.Connection( +def db_creds_root() -> Dict: + return dict( host=os.getenv("DJ_HOST"), user=os.getenv("DJ_USER"), password=os.getenv("DJ_PASS"), ) + + +@pytest.fixture(scope="session") +def connection_root_bare(db_creds_root): + connection = dj.Connection(**db_creds_root) yield connection From 156d22968dc80df9d23e599dd0900f91979d66f3 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 10:31:18 -0700 Subject: [PATCH 5/9] First pass at migrating test_privileges --- tests/test_privileges.py | 113 ++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 55 deletions(-) diff --git a/tests/test_privileges.py b/tests/test_privileges.py index 0cb807a6b..23e1dc327 100644 --- a/tests/test_privileges.py +++ b/tests/test_privileges.py @@ -1,21 +1,74 @@ import os import pytest -import importlib import datajoint as dj from . import schema, CONN_INFO_ROOT, PREFIX from . import schema_privileges namespace = locals() +USER = "djsubset" + + +@pytest.fixture +def schema_priv(connection_test): + schema_priv = dj.Schema( + # PREFIX + "_schema_privileges", + context=schema_privileges.LOCALS_PRIV, + connection=connection_test, + ) + schema_priv(schema_privileges.Parent) + schema_priv(schema_privileges.Child) + schema_priv(schema_privileges.NoAccess) + schema_priv(schema_privileges.NoAccessAgain) + yield schema_priv + if schema_priv.is_activated(): + schema_priv.drop() + + +@pytest.fixture +def connection_djsubset(connection_root, db_creds_root, schema_priv): + user = "djsubset" + conn = dj.conn(**db_creds_root, reset=True) + schema_priv.activate(f"{PREFIX}_schema_privileges") + conn.query( + f""" + CREATE USER IF NOT EXISTS '{user}'@'%%' + IDENTIFIED BY '{user}' + """ + ) + conn.query( + f""" + GRANT SELECT, INSERT, UPDATE, DELETE + ON `{PREFIX}_schema_privileges`.`#parent` + TO '{user}'@'%%' + """ + ) + conn.query( + f""" + GRANT SELECT, INSERT, UPDATE, DELETE + ON `{PREFIX}_schema_privileges`.`__child` + TO '{user}'@'%%' + """ + ) + conn_djsubset = dj.conn( + host=db_creds_root["host"], + user=user, + password=user, + reset=True, + ) + yield conn_djsubset + conn.query(f"DROP USER {user}") + conn.query(f"DROP DATABASE {PREFIX}_schema_privileges") + @pytest.fixture -def connection_djview(connection_root): +def connection_djview(connection_root, db_creds_root): """ A connection with only SELECT privilege to djtest schemas. Requires connection_root fixture so that `djview` user exists. """ connection = dj.conn( - host=os.getenv("DJ_HOST"), + host=db_creds_root["host"], user="djview", password="djview", reset=True, @@ -60,58 +113,8 @@ class Try(dj.Manual): class TestSubset: - USER = "djsubset" - - @classmethod - def setup_class(cls): - conn = dj.conn( - host=CONN_INFO_ROOT["host"], - user=CONN_INFO_ROOT["user"], - password=CONN_INFO_ROOT["password"], - reset=True, - ) - schema_privileges.schema.activate(f"{PREFIX}_schema_privileges") - conn.query( - f""" - CREATE USER IF NOT EXISTS '{cls.USER}'@'%%' - IDENTIFIED BY '{cls.USER}' - """ - ) - conn.query( - f""" - GRANT SELECT, INSERT, UPDATE, DELETE - ON `{PREFIX}_schema_privileges`.`#parent` - TO '{cls.USER}'@'%%' - """ - ) - conn.query( - f""" - GRANT SELECT, INSERT, UPDATE, DELETE - ON `{PREFIX}_schema_privileges`.`__child` - TO '{cls.USER}'@'%%' - """ - ) - cls.connection = dj.conn( - host=CONN_INFO_ROOT["host"], - user=cls.USER, - password=cls.USER, - reset=True, - ) - - @classmethod - def teardown_class(cls): - conn = dj.conn( - host=CONN_INFO_ROOT["host"], - user=CONN_INFO_ROOT["user"], - password=CONN_INFO_ROOT["password"], - reset=True, - ) - conn.query(f"DROP USER {cls.USER}") - conn.query(f"DROP DATABASE {PREFIX}_schema_privileges") - - def test_populate_activate(self): - importlib.reload(schema_privileges) - schema_privileges.schema.activate( + def test_populate_activate(self, connection_djsubset, schema_priv): + schema_priv.activate( f"{PREFIX}_schema_privileges", create_schema=True, create_tables=False ) schema_privileges.Child.populate() From 05b3b002495738aee74c36341cd7ca90b88eb6d4 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 10:45:46 -0700 Subject: [PATCH 6/9] Clean up --- tests/test_privileges.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_privileges.py b/tests/test_privileges.py index 23e1dc327..4670e8e81 100644 --- a/tests/test_privileges.py +++ b/tests/test_privileges.py @@ -6,8 +6,6 @@ namespace = locals() -USER = "djsubset" - @pytest.fixture def schema_priv(connection_test): From 9b0df13058ab5fcfaeb2fa5f3723e0fdcd50824e Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 11:03:02 -0700 Subject: [PATCH 7/9] rm commented code --- tests/test_privileges.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_privileges.py b/tests/test_privileges.py index 4670e8e81..949dbc8aa 100644 --- a/tests/test_privileges.py +++ b/tests/test_privileges.py @@ -10,7 +10,6 @@ @pytest.fixture def schema_priv(connection_test): schema_priv = dj.Schema( - # PREFIX + "_schema_privileges", context=schema_privileges.LOCALS_PRIV, connection=connection_test, ) From 394cad97a9a10e6732038d8c822d98baf528f583 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 11:24:07 -0700 Subject: [PATCH 8/9] Add default values for db_creds_root --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 470c2f440..249080601 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,9 +55,9 @@ def enable_filepath_feature(monkeypatch): @pytest.fixture(scope="session") def db_creds_root() -> Dict: return dict( - host=os.getenv("DJ_HOST"), - user=os.getenv("DJ_USER"), - password=os.getenv("DJ_PASS"), + host=os.getenv("DJ_HOST", "fakeservices.datajoint.io"), + user=os.getenv("DJ_USER", "root"), + password=os.getenv("DJ_PASS", "password"), ) From 36623f1cb7fcd25429d064c8788648e781fc0949 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Wed, 13 Dec 2023 12:39:12 -0700 Subject: [PATCH 9/9] Format with black --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 249080601..d4e4a23c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,7 @@ def enable_filepath_feature(monkeypatch): @pytest.fixture(scope="session") def db_creds_root() -> Dict: return dict( - host=os.getenv("DJ_HOST", "fakeservices.datajoint.io"), + host=os.getenv("DJ_HOST", "fakeservices.datajoint.io"), user=os.getenv("DJ_USER", "root"), password=os.getenv("DJ_PASS", "password"), )