diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3469f200..247809db 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.10", "3.9", "3.8", "3.7"] + python-version: ["3.12", "3.11", "3.10", "3.9", "3.8", "3.7"] neo4j-version: ["community", "enterprise", "5.5-enterprise", "4.4-enterprise", "4.4-community"] steps: diff --git a/Changelog b/Changelog index 4fefdc0f..69be5da2 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,9 @@ +Version 5.2.1 2023-12 +* Add options to inspection script to skip heavy operations - rel props or cardinality inspection #767 +* Fixes database version parsing issues +* Fixes bug when combining count with pagination #769 +* Bumps neo4j (driver) to 5.15.0 + Version 5.2.0 2023-11 * Add an option to pass your own driver instead of relying on the automatically created one. See set_connection method. NB : only accepts the synchronous driver for now. * Add a close_connection method to explicitly close the driver to match Neo4j deprecation. diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index d5299f54..ed1af11b 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -32,7 +32,7 @@ Adjust driver configuration - these options are only available for this connecti config.MAX_TRANSACTION_RETRY_TIME = 30.0 # default config.RESOLVER = None # default config.TRUST = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES # default - config.USER_AGENT = neomodel/v5.2.0 # default + config.USER_AGENT = neomodel/v5.2.1 # default Setting the database name, if different from the default one:: diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 4b832190..25e999e7 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -81,8 +81,18 @@ You can inspect an existing Neo4j database to generate a neomodel definition fil This will generate a file called ``models.py`` in the ``yourapp`` directory. This file can be used as a starting point, and will contain the necessary module imports, as well as class definition for nodes and, if relevant, relationships. +Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking +your credentials. + Note that you can also print the output to the console instead of writing a file by omitting the ``--write-to`` option. +If you have a database with a large number of nodes and relationships, +this script can take a long time to run (during our tests, it took 30 seconds for 500k nodes and 1.3M relationships). +You can speed it up by not scanning for relationship properties and/or relationship cardinality, using these options : +``--no-rel-props`` and ``--no-rel-cardinality``. +Note that this will still add relationship definition to your nodes, but without relationship models ; +and cardinality will be default (ZeroOrMore). + .. note:: This command will only generate the definition for nodes and relationships that are present in the @@ -108,6 +118,9 @@ script (:ref:`neomodel_install_labels`) to automate this: :: It is important to execute this after altering the schema and observe the number of classes it reports. +Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking +your credentials. + Remove existing constraints and indexes ======================================= Similarly, ``neomodel`` provides a script (:ref:`neomodel_remove_labels`) to automate the removal of all existing constraints and indexes from @@ -117,6 +130,9 @@ the database, when this is required: :: After executing, it will print all indexes and constraints it has removed. +Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking +your credentials. + Create, Update, Delete operations ================================= diff --git a/neomodel/__init__.py b/neomodel/__init__.py index 23e0142a..aee40a08 100644 --- a/neomodel/__init__.py +++ b/neomodel/__init__.py @@ -1,5 +1,4 @@ # pep8: noqa -import pkg_resources from neomodel.exceptions import * from neomodel.match import EITHER, INCOMING, OUTGOING, NodeSet, Traversal diff --git a/neomodel/_version.py b/neomodel/_version.py index 6c235c59..98886d26 100644 --- a/neomodel/_version.py +++ b/neomodel/_version.py @@ -1 +1 @@ -__version__ = "5.2.0" +__version__ = "5.2.1" diff --git a/neomodel/match.py b/neomodel/match.py index 2773d6e5..fb47f568 100644 --- a/neomodel/match.py +++ b/neomodel/match.py @@ -323,6 +323,7 @@ class QueryAST: result_class: Optional[type] lookup: Optional[str] additional_return: Optional[list] + is_count: Optional[bool] def __init__( self, @@ -337,6 +338,7 @@ def __init__( result_class: Optional[type] = None, lookup: Optional[str] = None, additional_return: Optional[list] = None, + is_count: Optional[bool] = False, ): self.match = match if match else [] self.optional_match = optional_match if optional_match else [] @@ -349,6 +351,7 @@ def __init__( self.result_class = result_class self.lookup = lookup self.additional_return = additional_return if additional_return else [] + self.is_count = is_count class QueryBuilder: @@ -649,15 +652,27 @@ def build_query(self): query += " ORDER BY " query += ", ".join(self._ast.order_by) - if self._ast.skip: + # If we return a count with pagination, pagination has to happen before RETURN + # It will then be included in the WITH clause already + if self._ast.skip and not self._ast.is_count: query += f" SKIP {self._ast.skip}" - if self._ast.limit: + if self._ast.limit and not self._ast.is_count: query += f" LIMIT {self._ast.limit}" return query def _count(self): + self._ast.is_count = True + # If we return a count with pagination, pagination has to happen before RETURN + # Like : WITH my_var SKIP 10 LIMIT 10 RETURN count(my_var) + self._ast.with_clause = f"{self._ast.return_clause}" + if self._ast.skip: + self._ast.with_clause += f" SKIP {self._ast.skip}" + + if self._ast.limit: + self._ast.with_clause += f" LIMIT {self._ast.limit}" + self._ast.return_clause = f"count({self._ast.return_clause})" # drop order_by, results in an invalid query self._ast.order_by = None diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 1a24675a..a8cf4c0e 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -19,11 +19,13 @@ If no file is specified, the tool will print the class definitions to stdout. options: - -h, --help show this help message and exit - --db bolt://neo4j:neo4j@localhost:7687 + -h, --help show this help message and exit + --db bolt://neo4j:neo4j@localhost:7687 Neo4j Server URL - -T, --write-to someapp/models.py + -T, --write-to someapp/models.py File where to write output. + --no-rel-props Do not inspect relationship properties + --no-rel-cardinality Do not infer relationship cardinality """ import argparse @@ -116,13 +118,20 @@ def get_indexed_properties_for_label(label): class RelationshipInspector: @classmethod - def outgoing_relationships(cls, start_label): - query = f""" - MATCH (n:`{start_label}`)-[r]->(m) - WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label, keys(r) AS properties, head(collect(r)) AS sampleRel - ORDER BY size(properties) DESC - RETURN rel_type, target_label, apoc.meta.cypher.types(properties(sampleRel)) AS properties LIMIT 1 - """ + def outgoing_relationships(cls, start_label, get_properties: bool = True): + if get_properties: + query = f""" + MATCH (n:`{start_label}`)-[r]->(m) + WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label, keys(r) AS properties, head(collect(r)) AS sampleRel + ORDER BY size(properties) DESC + RETURN rel_type, target_label, apoc.meta.cypher.types(properties(sampleRel)) AS properties LIMIT 1 + """ + else: + query = f""" + MATCH (n:`{start_label}`)-[r]->(m) + WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label + RETURN rel_type, target_label, {{}} AS properties LIMIT 1 + """ result, _ = db.cypher_query(query) return [(record[0], record[1], record[2]) for record in result] @@ -222,7 +231,9 @@ def parse_imports(): return imports -def build_rel_type_definition(label, outgoing_relationships, defined_rel_types): +def build_rel_type_definition( + label, outgoing_relationships, defined_rel_types, infer_cardinality: bool = True +): class_definition_append = "" rel_type_definitions = "" @@ -241,9 +252,12 @@ def build_rel_type_definition(label, outgoing_relationships, defined_rel_types): rel_type ) - cardinality = RelationshipInspector.infer_cardinality(rel_type, label) + cardinality_string = "" + if infer_cardinality: + cardinality = RelationshipInspector.infer_cardinality(rel_type, label) + cardinality_string += f", cardinality={cardinality}" - class_definition_append += f' {clean_class_member_key(rel_name)} = RelationshipTo("{target_label}", "{rel_type}", cardinality={cardinality}' + class_definition_append += f' {clean_class_member_key(rel_name)} = RelationshipTo("{target_label}", "{rel_type}"{cardinality_string}' if rel_props and rel_type not in defined_rel_types: rel_model_name = generate_rel_class_name(rel_type) @@ -265,7 +279,11 @@ def build_rel_type_definition(label, outgoing_relationships, defined_rel_types): return class_definition_append -def inspect_database(bolt_url): +def inspect_database( + bolt_url, + get_relationship_properties: bool = True, + infer_relationship_cardinality: bool = True, +): # Connect to the database print(f"Connecting to {bolt_url}") db.set_connection(bolt_url) @@ -284,23 +302,32 @@ def inspect_database(bolt_url): indexed_properties = NodeInspector.get_indexed_properties_for_label(label) class_definition = f"class {class_name}(StructuredNode):\n" - class_definition += "".join( - [ - build_prop_string( - unique_properties, indexed_properties, prop, prop_type - ) - for prop, prop_type in properties.items() - ] - ) + if properties: + class_definition += "".join( + [ + build_prop_string( + unique_properties, indexed_properties, prop, prop_type + ) + for prop, prop_type in properties.items() + ] + ) - outgoing_relationships = RelationshipInspector.outgoing_relationships(label) + outgoing_relationships = RelationshipInspector.outgoing_relationships( + label, get_relationship_properties + ) if outgoing_relationships and "StructuredRel" not in IMPORTS: IMPORTS.append("RelationshipTo") - IMPORTS.append("StructuredRel") + # No rel properties = no rel classes + # Then StructuredRel import is not needed + if get_relationship_properties: + IMPORTS.append("StructuredRel") class_definition += build_rel_type_definition( - label, outgoing_relationships, defined_rel_types + label, + outgoing_relationships, + defined_rel_types, + infer_relationship_cardinality, ) if not properties and not outgoing_relationships: @@ -353,6 +380,20 @@ def main(): help="File where to write output.", ) + parser.add_argument( + "--no-rel-props", + dest="get_relationship_properties", + action="store_false", + help="Do not inspect relationship properties", + ) + + parser.add_argument( + "--no-rel-cardinality", + dest="infer_relationship_cardinality", + action="store_false", + help="Do not infer relationship cardinality", + ) + args = parser.parse_args() bolt_url = args.neo4j_bolt_url @@ -364,12 +405,22 @@ def main(): # Before connecting to the database if args.write_to: with open(args.write_to, "w") as file: - output = inspect_database(bolt_url=bolt_url) + output = inspect_database( + bolt_url=bolt_url, + get_relationship_properties=args.get_relationship_properties, + infer_relationship_cardinality=args.infer_relationship_cardinality, + ) print(f"Writing to {args.write_to}") file.write(output) # If no file is specified, print to stdout else: - print(inspect_database(bolt_url=bolt_url)) + print( + inspect_database( + bolt_url=bolt_url, + get_relationship_properties=args.get_relationship_properties, + infer_relationship_cardinality=args.infer_relationship_cardinality, + ) + ) if __name__ == "__main__": diff --git a/neomodel/util.py b/neomodel/util.py index cf4230e3..74e88250 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -679,9 +679,9 @@ def version_tag_to_integer(version_tag): """ Converts a version string to an integer representation to allow for quick comparisons between versions. - :param a_version_string: The version string to be converted (e.g. '3.4.0') + :param a_version_string: The version string to be converted (e.g. '5.4.0') :type a_version_string: str - :return: An integer representation of the version string (e.g. '3.4.0' --> 340) + :return: An integer representation of the version string (e.g. '5.4.0' --> 50400) :rtype: int """ components = version_tag.split(".") @@ -689,5 +689,9 @@ def version_tag_to_integer(version_tag): components.append("0") num = 0 for index, component in enumerate(components): - num += (10 ** ((len(components) - 1) - index)) * int(component) + # Aura started adding a -aura suffix in version numbers, like "5.14-aura" + # This will strip the suffix to allow for proper comparison : 14 instead of 14-aura + if "-" in component: + component = component.split("-")[0] + num += (100 ** ((len(components) - 1) - index)) * int(component) return num diff --git a/pyproject.toml b/pyproject.toml index c5cda7e6..f15c28b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,25 +1,15 @@ -[build-system] -requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] -build-backend = "setuptools.build_meta" - -[tool.setuptools_scm] - -[tool.setuptools.packages.find] -where = ["./"] - [project] name = "neomodel" authors = [ {name = "Robin Edwards", email = "robin.ge@gmail.com"}, ] maintainers = [ + {name = "Marius Conjeaud", email = "marius.conjeaud@outlook.com"}, {name = "Athanasios Anastasiou", email = "athanastasiou@gmail.com"}, {name = "Cristina Escalante"}, - {name = "Marius Conjeaud", email = "marius.conjeaud@outlook.com"}, ] description = "An object mapper for the neo4j graph database." readme = "README.md" -requires-python = ">=3.7" keywords = ["graph", "neo4j", "ORM", "OGM", "mapper"] license = {text = "MIT"} classifiers = [ @@ -33,12 +23,10 @@ classifiers = [ "Topic :: Database", ] dependencies = [ - "neo4j==5.12.0", - "pytz>=2021.1", - "neobolt==1.7.17", - "six==1.16.0", + "neo4j~=5.15.0", ] -version='5.2.0' +requires-python = ">=3.7" +dynamic = ["version"] [project.urls] documentation = "https://neomodel.readthedocs.io/en/latest/" @@ -57,6 +45,16 @@ dev = [ pandas = ["pandas"] numpy = ["numpy"] +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.dynamic] +version = {attr = "neomodel._version.__version__"} + +[tool.setuptools.packages.find] +where = ["./"] + [tool.pytest.ini_options] addopts = "--resetdb" testpaths = "test" diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..2e7a31f0 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,9 @@ +# neomodel +-e .[pandas,numpy] + +pytest>=7.1 +pytest-cov>=4.0 +pre-commit +black +isort +Shapely>=2.0.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9e91cbc0..79a520c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1 @@ -neo4j-driver==4.4.10 -pytz>=2021.1 -neobolt==1.7.17 -six==1.16.0 -sphinx-copybutton +neo4j~=5.14.1 diff --git a/test/data/neomodel_inspect_database_output_light.txt b/test/data/neomodel_inspect_database_output_light.txt new file mode 100644 index 00000000..c9adeca3 --- /dev/null +++ b/test/data/neomodel_inspect_database_output_light.txt @@ -0,0 +1,26 @@ +from neomodel import StructuredNode, StringProperty, RelationshipTo, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel.contrib.spatial_properties import PointProperty + +class ScriptsTestNode(StructuredNode): + personal_id = StringProperty(unique_index=True) + name = StringProperty(index=True) + rel = RelationshipTo("ScriptsTestNode", "REL") + + +class EveryPropertyTypeNode(StructuredNode): + array_property = ArrayProperty(StringProperty()) + float_property = FloatProperty() + boolean_property = BooleanProperty() + point_property = PointProperty(crs='wgs-84') + string_property = StringProperty() + datetime_property = DateTimeProperty() + integer_property = IntegerProperty() + + +class NoPropertyNode(StructuredNode): + pass + + +class NoPropertyRelNode(StructuredNode): + no_prop_rel = RelationshipTo("NoPropertyRelNode", "NO_PROP_REL") + diff --git a/test/data/neomodel_inspect_database_output_pre_5_7_light.txt b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt new file mode 100644 index 00000000..c9adeca3 --- /dev/null +++ b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt @@ -0,0 +1,26 @@ +from neomodel import StructuredNode, StringProperty, RelationshipTo, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel.contrib.spatial_properties import PointProperty + +class ScriptsTestNode(StructuredNode): + personal_id = StringProperty(unique_index=True) + name = StringProperty(index=True) + rel = RelationshipTo("ScriptsTestNode", "REL") + + +class EveryPropertyTypeNode(StructuredNode): + array_property = ArrayProperty(StringProperty()) + float_property = FloatProperty() + boolean_property = BooleanProperty() + point_property = PointProperty(crs='wgs-84') + string_property = StringProperty() + datetime_property = DateTimeProperty() + integer_property = IntegerProperty() + + +class NoPropertyNode(StructuredNode): + pass + + +class NoPropertyRelNode(StructuredNode): + no_prop_rel = RelationshipTo("NoPropertyRelNode", "NO_PROP_REL") + diff --git a/test/test_dbms_awareness.py b/test/test_dbms_awareness.py index fa041afe..93dc032d 100644 --- a/test/test_dbms_awareness.py +++ b/test/test_dbms_awareness.py @@ -1,6 +1,7 @@ from pytest import mark from neomodel import db +from neomodel.util import version_tag_to_integer @mark.skipif( @@ -9,6 +10,7 @@ def test_version_awareness(): assert db.database_version == "5.7.0" assert db.version_is_higher_than("5.7") + assert db.version_is_higher_than("5.6.0") assert db.version_is_higher_than("5") assert db.version_is_higher_than("4") @@ -20,3 +22,11 @@ def test_edition_awareness(): assert db.edition_is_enterprise() else: assert not db.edition_is_enterprise() + + +def test_version_tag_to_integer(): + assert version_tag_to_integer("5.7.1") == 50701 + assert version_tag_to_integer("5.1") == 50100 + assert version_tag_to_integer("5") == 50000 + assert version_tag_to_integer("5.14.1") == 51401 + assert version_tag_to_integer("5.14-aura") == 51400 diff --git a/test/test_label_install.py b/test/test_label_install.py index 69c0f7af..46f55467 100644 --- a/test/test_label_install.py +++ b/test/test_label_install.py @@ -1,5 +1,4 @@ import pytest -from six import StringIO from neomodel import ( RelationshipTo, @@ -118,11 +117,11 @@ class OtherNodeWithUniqueIndexRelationship(StructuredNode): assert expected_std_out in captured.out -def test_install_labels_db_property(): - stdout = StringIO() +def test_install_labels_db_property(capsys): drop_constraints() - install_labels(SomeNotUniqueNode, quiet=False, stdout=stdout) - assert "id" in stdout.getvalue() + install_labels(SomeNotUniqueNode, quiet=False) + captured = capsys.readouterr() + assert "id" in captured.out # make sure that the id_ constraint doesn't exist constraint_names = _drop_constraints_for_label_and_property( "SomeNotUniqueNode", "id_" diff --git a/test/test_match_api.py b/test/test_match_api.py index 43c7a104..ee6b337e 100644 --- a/test/test_match_api.py +++ b/test/test_match_api.py @@ -140,6 +140,13 @@ def test_count(): count = QueryBuilder(NodeSet(source=Coffee)).build_ast()._count() assert count > 0 + Coffee(name="Kawa", price=27).save() + node_set = NodeSet(source=Coffee) + node_set.skip = 1 + node_set.limit = 1 + count = QueryBuilder(node_set).build_ast()._count() + assert count == 1 + def test_len_and_iter_and_bool(): iterations = 0 diff --git a/test/test_scripts.py b/test/test_scripts.py index 66594489..e30fd213 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -1,5 +1,7 @@ import subprocess +import pytest + from neomodel import ( RelationshipTo, StringProperty, @@ -87,7 +89,15 @@ def test_neomodel_remove_labels(): assert len(indexes) == 0 -def test_neomodel_inspect_database(): +@pytest.mark.parametrize( + "script_flavour", + [ + "", + "_light", + ], +) +def test_neomodel_inspect_database(script_flavour): + output_file = "test/data/neomodel_inspect_database_test_output.py" # Check that the help option works result = subprocess.run( ["neomodel_inspect_database", "--help"], @@ -128,8 +138,11 @@ def test_neomodel_inspect_database(): ) # Test the console output version of the script + args_list = ["neomodel_inspect_database", "--db", config.DATABASE_URL] + if script_flavour == "_light": + args_list += ["--no-rel-props", "--no-rel-cardinality"] result = subprocess.run( - ["neomodel_inspect_database", "--db", config.DATABASE_URL], + args_list, capture_output=True, text=True, check=True, @@ -141,9 +154,9 @@ def test_neomodel_inspect_database(): assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here file_path = ( - "test/data/neomodel_inspect_database_output.txt" + f"test/data/neomodel_inspect_database_output{script_flavour}.txt" if db.version_is_higher_than("5.7") - else "test/data/neomodel_inspect_database_output_pre_5_7.txt" + else f"test/data/neomodel_inspect_database_output_pre_5_7{script_flavour}.txt" ) with open(file_path, "r") as f: wrapped_test_file = [line for line in f.read().split("\n") if line.strip()] @@ -165,14 +178,9 @@ def test_neomodel_inspect_database(): assert set(wrapped_test_file) == set(wrapped_console_output[2:]) # Test the file output version of the script + args_list += ["--write-to", output_file] result = subprocess.run( - [ - "neomodel_inspect_database", - "--db", - config.DATABASE_URL, - "--write-to", - "test/data/neomodel_inspect_database_test_output.py", - ], + args_list, capture_output=True, text=True, check=True, @@ -186,11 +194,11 @@ def test_neomodel_inspect_database(): ] assert wrapped_file_console_output[0].startswith("Connecting to") assert wrapped_file_console_output[1].startswith("Writing to") - with open("test/data/neomodel_inspect_database_test_output.py", "r") as f: + with open(output_file, "r") as f: wrapped_output_file = [line for line in f.read().split("\n") if line.strip()] assert set(wrapped_output_file) == set(wrapped_console_output[1:]) # Finally, delete the file created by the script subprocess.run( - ["rm", "test/data/neomodel_inspect_database_test_output.py"], + ["rm", output_file], )