diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 53941892e..000000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd \ No newline at end of file diff --git a/docs/source/keys.rst b/docs/source/keys.rst new file mode 100644 index 000000000..b4b34892f --- /dev/null +++ b/docs/source/keys.rst @@ -0,0 +1,66 @@ +================ +Keys in Snovault +================ + +Broadly speaking in the 4DN space there are four different types of 'keys'. 'Unique', 'name' and 'identifying' keys are used in both Fourfront/CGAP and Snovault while 'lookup' keys are used only in FF/CGAP. This document will only touch on the first three. Below is a small table illustrating where each type of key is defined. + + +.. csv-table:: + :header: "Schema", "Collection", "Type" + :widths: 10, 20, 10 + + uniqueKey, identification_key, name_key + + +Unique Key and Identification Key +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These two keys are bundled together because they are closely related. A unique key is denoted in the schema of an item. This is a constraint on the items in the database that says that no two items can share the same value in this field. Note that if there are multiple fields that are marked as uniqueKey's then either one can be used to uniquely identify items. In either case a identification key must be specified in the collection decorator, which must be one of the unique keys. If a identification key is not specified you will not be able to lookup items via the resource path with either of their unique keys. See the below examples from the tests. + +.. code-block:: python + + # All of the following collections are referencing a schema which denotes + # fields 'obj_id' and 'name' as uniqueKey's. In the below case though if you + # wanted to look up an item by this type from the point of view of snovault + # you would only be able to do so via uuid + @collection('testing-keys') + class TestingKeys(Item): + """ Intended to test the behavior of uniqueKey value in schema """ + item_type = 'testing_keys' + schema = load_schema('snovault:test_schemas/TestingKeys.json') + + + # In this case we specify the identification_key to be the obj_id. This allows us to + # use the resource path to get the item ie: Get /testing-keys-def/ + # Note that the resource path is still the uuid + @collection('testing-keys-def', identification_key='testing_keys_def:obj_id') + class TestingKeysDef(Item): + """ + Intended to test the behavior of setting a identification key equal to one of the + uniqueKey's specified in the schema. This should allow us to get the object + via obj_id whereas before we could not. + """ + item_type = 'testing_keys_def' + schema = load_schema('snovault:test_schemas/TestingKeys.json') + +Name Key +^^^^^^^^ + +The name key is a special field specified on the item type definition. It augments the resource path so that the '@id' field of the item contains a path using the name_key instead of the uuid. To be explicit, the name key must match the identification key and is only really useful for changing the resource path. See final example below. + +.. code-block:: python + + # In this case we specify matching identification_key and name_key. This means that + # the resource path is augmented to show the name_key instead of the uuid AND + # you can get the item via resource path ie: Get /testing-keys-name/ + # and that is what always shows up when you access that item + @collection('testing-keys-name', identification_key='testing_keys_name:name') + class TestingKeysName(Item): + """ + We set name as a identification key so that it can be used as a name_key in the + resource path. We should now see the name key in the @id field instead of + the uuid + """ + item_type = 'testing_keys_name' + schema = load_schema('snovault:test_schemas/TestingKeys.json') + name_key = 'name' diff --git a/src/snovault/_version.py b/src/snovault/_version.py index cbda5a75f..70dcf2ade 100644 --- a/src/snovault/_version.py +++ b/src/snovault/_version.py @@ -1,4 +1,4 @@ """Version information.""" # The following line *must* be the last in the module, exactly as formatted: -__version__ = "1.3.4" +__version__ = "1.3.6" diff --git a/src/snovault/elasticsearch/create_mapping.py b/src/snovault/elasticsearch/create_mapping.py index 1e5eb0b78..04f38376d 100644 --- a/src/snovault/elasticsearch/create_mapping.py +++ b/src/snovault/elasticsearch/create_mapping.py @@ -474,11 +474,6 @@ def es_mapping(mapping, agg_items_mapping): 'type': 'object', 'include_in_all': False }, - 'paths': { - 'type': 'keyword', - 'ignore_above': KW_IGNORE_ABOVE, - 'include_in_all': False - }, 'indexing_stats': { 'type': 'object', 'include_in_all': False diff --git a/src/snovault/elasticsearch/indexer_utils.py b/src/snovault/elasticsearch/indexer_utils.py index b282e0b91..3a4dc49b6 100644 --- a/src/snovault/elasticsearch/indexer_utils.py +++ b/src/snovault/elasticsearch/indexer_utils.py @@ -12,7 +12,7 @@ def get_namespaced_index(config, index): settings = config.registry.settings except: # accept either config or registry as first arg settings = config.settings - namespace = settings.get('indexer.namespace', '') + namespace = settings.get('indexer.namespace') or '' return namespace + index diff --git a/src/snovault/indexing_views.py b/src/snovault/indexing_views.py index 5f8cca522..1aae44249 100644 --- a/src/snovault/indexing_views.py +++ b/src/snovault/indexing_views.py @@ -106,28 +106,11 @@ def item_index_data(context, request): links = new_links principals_allowed = calc_principals(context) - path = resource_path(context) - paths = {path} + path = resource_path(context) + '/' collection = context.collection - with indexing_timer(indexing_stats, 'unique_keys'): unique_keys = context.unique_keys(properties) - if collection.unique_key in unique_keys: - paths.update( - resource_path(collection, key) - for key in unique_keys[collection.unique_key]) - - with indexing_timer(indexing_stats, 'paths'): - for base in (collection, request.root): - for key_name in ('accession', 'alias'): - if key_name not in unique_keys: - continue - paths.add(resource_path(base, uuid)) - paths.update( - resource_path(base, key) - for key in unique_keys[key_name]) - - path = path + '/' + # setting _indexing_view enables the embed_cache and cause population of # request._linked_uuids and request._rev_linked_uuids_by_item request._indexing_view = True @@ -182,7 +165,6 @@ def item_index_data(context, request): 'links': links, 'max_sid': context.max_sid, 'object': object_view, - 'paths': sorted(paths), 'principals_allowed': principals_allowed, 'properties': properties, 'propsheets': { diff --git a/src/snovault/resources.py b/src/snovault/resources.py index 912b25260..dcb7bcd91 100644 --- a/src/snovault/resources.py +++ b/src/snovault/resources.py @@ -76,7 +76,7 @@ def jsonld_context(self, request): def actions(self, request): actions = calculate_properties(self, request, category='action') if actions: - return list(actions.values()) + return sorted(list(actions.values()), key=lambda d: d.get('name')) class Root(Resource): @@ -142,12 +142,12 @@ class AbstractCollection(Resource, Mapping): And some other info as well. Collections allow retrieval of specific items with them by using the `get` - method with uuid or the unique_key + method with uuid or the identification_key (which must be a unique key) """ properties = {} - unique_key = None + identification_key = None - def __init__(self, registry, name, type_info, properties=None, acl=None, unique_key=None): + def __init__(self, registry, name, type_info, properties=None, acl=None, identification_key=None): self.registry = registry self.__name__ = name self.type_info = type_info @@ -155,8 +155,8 @@ def __init__(self, registry, name, type_info, properties=None, acl=None, unique_ self.properties = properties if acl is not None: self.__acl__ = acl - if unique_key is not None: - self.unique_key = unique_key + if identification_key is not None: + self.identification_key = identification_key @reify def connection(self): @@ -200,8 +200,8 @@ def get(self, name, default=None): if not self._allow_contained(resource): return default return resource - if self.unique_key is not None: - resource = self.connection.get_by_unique_key(self.unique_key, name) + if self.identification_key is not None: + resource = self.connection.get_by_unique_key(self.identification_key, name) if resource is not None: if not self._allow_contained(resource): return default @@ -248,8 +248,8 @@ class Collection(AbstractCollection): class Item(Resource): item_type = None - base_types = ['Item'] name_key = None + base_types = ['Item'] rev = {} aggregated_items = {} embedded_list = [] @@ -280,7 +280,6 @@ def __parent__(self): @property def __name__(self): - if self.name_key is None: return str(self.uuid) return self.properties.get(self.name_key, None) or str(self.uuid) @@ -359,6 +358,7 @@ def get_filtered_rev_links(self, request, name): return filtered_uuids def unique_keys(self, properties): + """ Gets all schema fields defined to be uniqueKey's """ return { name: [v for prop in props for v in ensurelist(properties.get(prop, ()))] for name, props in self.type_info.schema_keys.items() diff --git a/src/snovault/test_schemas/EmbeddingTest.json b/src/snovault/test_schemas/EmbeddingTest.json index 17aaa825b..c7a2ed8ed 100644 --- a/src/snovault/test_schemas/EmbeddingTest.json +++ b/src/snovault/test_schemas/EmbeddingTest.json @@ -33,4 +33,4 @@ "title": "Date added" } } -} \ No newline at end of file +} diff --git a/src/snovault/test_schemas/TestingKeys.json b/src/snovault/test_schemas/TestingKeys.json new file mode 100644 index 000000000..25fcde991 --- /dev/null +++ b/src/snovault/test_schemas/TestingKeys.json @@ -0,0 +1,27 @@ +{ + "type": "object", + "properties": { + "name": { + "title": "Common Name", + "description": "Unique name for this object", + "type": "string", + "uniqueKey": true + }, + "grouping": { + "title": "Grouping", + "description": "String name of a group that this item belongs to (not unique)", + "type": "string" + }, + "obj_id": { + "title": "Object ID", + "description": "Unique ID of this object", + "type": "string", + "uniqueKey": true + }, + "system_id": { + "title": "System ID", + "description": "Unique System ID for this object that is not marked as unique in schema", + "type": "string" + } + } +} diff --git a/src/snovault/test_schemas/TestingLinkTargetSno.json b/src/snovault/test_schemas/TestingLinkTargetSno.json index 4e7f68610..b7df15aaa 100644 --- a/src/snovault/test_schemas/TestingLinkTargetSno.json +++ b/src/snovault/test_schemas/TestingLinkTargetSno.json @@ -13,4 +13,4 @@ } }, "additionalProperties": false -} \ No newline at end of file +} diff --git a/src/snovault/tests/test_keyed_items.py b/src/snovault/tests/test_keyed_items.py new file mode 100644 index 000000000..d9f778280 --- /dev/null +++ b/src/snovault/tests/test_keyed_items.py @@ -0,0 +1,148 @@ +import pytest + + +@pytest.fixture +def TestKey(testapp): + """ Posts an item under testing_keys_schema """ + url = '/testing-keys' + item = { + 'name': 'Orange', + 'grouping': 'fruit', + 'obj_id': '123' + } + testapp.post_json(url, item, status=201) + + +@pytest.fixture +def TestKeyDefinition(testapp): + """ Posts an item under testing_keys_def """ + url = '/testing-keys-def' + item = { + 'name': 'Orange', + 'grouping': 'fruit', + 'obj_id': '123', + 'system_id': 'abc' + } + testapp.post_json(url, item, status=201) + + +@pytest.fixture +def TestKeyName(testapp): + """ Posts an item under testing_keys_name """ + url = '/testing-keys-name' + item = { + 'name': 'Orange', + 'grouping': 'fruit', + 'obj_id': '123', + 'system_id': 'abc' + } + testapp.post_json(url, item, status=201) + + +def test_schema_unique_key(TestKey, testapp): + """ + Tests that when we define a uniqueKey on the schema that we cannot post a + second item with a repeat key value in any fields + """ + url = '/testing-keys' + duplicate_name = { + 'name': 'Orange', + 'grouping': 'fruit', + 'obj_id': '456', + 'system_id': 'def' + } + testapp.post_json(url, duplicate_name, status=409) + duplicate_id = { + 'name': 'Banana', + 'grouping': 'fruit', + 'obj_id': '123', + 'system_id': 'def' + } + testapp.post_json(url, duplicate_id, status=409) + duplicate_system_id = { + 'name': 'Banana', + 'grouping': 'fruit', + 'obj_id': '456', + 'system_id': 'abc' + } # should be allowed since not marked as unique wrt the DB + testapp.post_json(url, duplicate_system_id, status=201) + correct = { + 'name': 'Apple', + 'grouping': 'fruit', + 'obj_id': '789', + 'system_id': 'hij' + } # should also work since all fields are unique + testapp.post_json(url, correct, status=201) + # both gets should not work since obj_id and name not are name_keys + testapp.get(url + '/' + correct['obj_id'], status=404) + testapp.get(url + '/' + correct['name'], status=404) + + +def test_definition_unique_key(TestKeyDefinition, testapp): + """ + Tests that using 'obj_id' as the name key allows lookup via resource path + using 'obj_id' and not 'name' + """ + url = '/testing-keys-def' + duplicate_id = { + 'name': 'Banana', + 'grouping': 'fruit', + 'obj_id': '123', + 'system_id': 'def' + } # sanity: posting should fail as above since obj_id is a uniqueKey + testapp.post_json(url, duplicate_id, status=409) + duplicate_system_id = { + 'name': 'Banana', + 'grouping': 'fruit', + 'obj_id': '456', + 'system_id': 'abc' + } # this will work despite system_id being marked as unique_key in definition + testapp.post_json(url, duplicate_system_id, status=201) + # obj_id should succeed since it is a unique_key + resp = testapp.get(url + '/' + duplicate_system_id['obj_id']).follow(status=200).json + # since obj_id is a identification_key but not a name_key we still expect uuid + assert resp['uuid'] in resp['@id'] + # should fail since name is not a name_key + testapp.get(url + '/' + duplicate_system_id['name'], status=404) + + +def test_name_key(TestKeyName, testapp): + """ + Tests that using 'name' as the name key allows lookup via resource path + using 'name' and not 'obj_id' + """ + url = '/testing-keys-name' + duplicate_id = { + 'name': 'Banana', + 'grouping': 'fruit', + 'obj_id': '123', + 'system_id': 'def' + } # sanity: posting should fail as above since obj_id is a uniqueKey + testapp.post_json(url, duplicate_id, status=409) + correct = { + 'name': 'Apple', + 'grouping': 'fruit', + 'obj_id': '789', + 'system_id': 'hij' + } + testapp.post_json(url, correct, status=201) + # obj_id should succeed since it is a name_key + resp = testapp.get(url + '/' + correct['name']).follow(status=200).json + assert resp['name'] in resp['@id'] # @id is name since its the name_key + testapp.get(url + '/' + correct['obj_id'], status=404) + + +def test_name_key_identification_key_mismatch(testapp): + """ + Tries to post an item to a type with mismatched traversal and name key + which will fail + """ + url = '/testing-keys-mismatch' + item = { + 'name': 'Orange', + 'grouping': 'fruit', + 'obj_id': '123', + 'system_id': 'abc' + } + with pytest.raises(KeyError): # error in embed.py:181 + testapp.post_json(url, item) diff --git a/src/snovault/tests/testing_key.py b/src/snovault/tests/testing_key.py index c0bf05191..23f35fb10 100644 --- a/src/snovault/tests/testing_key.py +++ b/src/snovault/tests/testing_key.py @@ -16,7 +16,7 @@ def includeme(config): 'title': 'Test keys', 'description': 'Testing. Testing. 1, 2, 3.', }, - unique_key='testing_accession', + identification_key='testing_accession', ) class TestingKey(Item): item_type = 'testing_key' diff --git a/src/snovault/tests/testing_views.py b/src/snovault/tests/testing_views.py index 56791c111..e3f84e6f7 100644 --- a/src/snovault/tests/testing_views.py +++ b/src/snovault/tests/testing_views.py @@ -275,7 +275,7 @@ def edit_json(context, request): @abstract_collection( name='abstractItemTests', - unique_key='accession', + identification_key='accession', properties={ 'title': "AbstractItemTests", 'description': "Abstract Item that is inherited for testing", @@ -288,7 +288,7 @@ class AbstractItemTest(Item): @collection( name='abstract-item-test-sub-items', - unique_key='accession', + identification_key='accession', properties={ 'title': "AbstractItemTestSubItems", 'description': "Item based off of AbstractItemTest" @@ -300,7 +300,7 @@ class AbstractItemTestSubItem(AbstractItemTest): @collection( name='abstract-item-test-second-sub-items', - unique_key='accession', + identification_key='accession', properties={ 'title': 'AbstractItemTestSecondSubItems', 'description': "Second item based off of AbstractItemTest" @@ -312,7 +312,7 @@ class AbstractItemTestSecondSubItem(AbstractItemTest): @collection( name='embedding-tests', - unique_key='accession', + identification_key='accession', properties={ 'title': 'EmbeddingTests', 'description': 'Listing of EmbeddingTests' @@ -339,7 +339,7 @@ class TestingDownload(ItemWithAttachment): schema = load_schema('snovault:test_schemas/TestingDownload.json') -@collection('testing-link-sources-sno', unique_key='testing_link_sources-sno:name') +@collection('testing-link-sources-sno', identification_key='testing_link_sources-sno:name') class TestingLinkSourceSno(Item): item_type = 'testing_link_source_sno' schema = load_schema('snovault:test_schemas/TestingLinkSourceSno.json') @@ -354,11 +354,11 @@ class TestingLinkAggregateSno(Item): } -@collection('testing-link-targets-sno', unique_key='testing_link_target_sno:name') +@collection('testing-link-targets-sno', identification_key='testing_link_target_sno:name') class TestingLinkTargetSno(Item): item_type = 'testing_link_target_sno' - name_key = 'name' schema = load_schema('snovault:test_schemas/TestingLinkTargetSno.json') + name_key = 'name' rev = { 'reverse': ('TestingLinkSourceSno', 'target'), } @@ -410,6 +410,48 @@ class TestingDependencies(Item): schema = load_schema('snovault:test_schemas/TestingDependencies.json') +@collection('testing-keys') +class TestingKeys(Item): + """ Intended to test the behavior of uniqueKey value in schema """ + item_type = 'testing_keys' + schema = load_schema('snovault:test_schemas/TestingKeys.json') + + +@collection('testing-keys-def', identification_key='testing_keys_def:obj_id') +class TestingKeysDef(Item): + """ + Intended to test the behavior of setting a traversal key equal to one of the + uniqueKey's specified in the schema. This should allow us to get the object + via obj_id whereas before we could not. + """ + item_type = 'testing_keys_def' + schema = load_schema('snovault:test_schemas/TestingKeys.json') + + +@collection('testing-keys-name', identification_key='testing_keys_name:name') +class TestingKeysName(Item): + """ + We set name as a traversal key so that it can be used as a name_key in the + resource path. We should now see the name key in the @id field instead of + the uuid + """ + item_type = 'testing_keys_name' + schema = load_schema('snovault:test_schemas/TestingKeys.json') + name_key = 'name' + + +@collection('testing-keys-mismatch', identification_key='testing_keys_mismatch:name') +class TestingKeysMismatch(Item): + """ + Tests behavior when we set a traversal key to one value and name key to + another. In this case posting anything should fail since the traversal key + and name key must match. + """ + item_type = 'testing_keys_mismatch' + schema = load_schema('snovault:test_schemas/TestingKeys.json') + name_key = 'obj_id' + + @view_config(name='testing-render-error', request_method='GET') def testing_render_error(request): return {