Skip to content

Commit

Permalink
Merge pull request #108 from archesproject/cherry-pick-backend-fixes
Browse files Browse the repository at this point in the history 
Cherry pick backend fixes
  • Loading branch information
@chrabyrd
chrabyrd authored Oct 29, 2024
2 parents d84ae35 + 974709c commit a67c758
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 101 deletions.
10 changes: 3 additions & 7 deletions arches_lingo/const.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,6 @@
SCHEME_NAME_TYPE_NODE = "ef87b588-11de-11ef-9493-0a58a9feac02"


PREF_LABEL_VALUE_ID = "6ac8e471-476e-4fd0-b276-86e01a17bcc8"
ALT_LABEL_VALUE_ID = "d9e362ca-c155-4569-9197-5583dd66f7e5"
HIDDEN_LABEL_VALUE_ID = "18c46580-8c3c-48b7-9a6c-a0643708cb8b"

# Old RDM concepts, values
LANGUAGE_CONCEPT_ID = "a6b88323-7226-4428-8f41-3d5252e3a2a9"
ENGLISH_VALUE_ID = "de978fd0-2819-4855-8858-8c089780f32c"
### Lists and List Items ###
LANGUAGES_LIST_ID = "55ce793b-a51a-4b25-811d-d08ea797f8c3"
LABEL_LIST_ID = "deb847fc-f4c3-4e82-be19-04641579f129"
57 changes: 10 additions & 47 deletions arches_lingo/utils/concept_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,11 @@

from django.contrib.postgres.expressions import ArraySubquery
from django.core.cache import caches
from django.db.models import CharField, Exists, F, OuterRef, Value
from django.db.models import CharField, F, OuterRef, Value
from django.db.models.expressions import CombinedExpression
from django.utils.translation import gettext_lazy as _

from arches.app.models.models import (
Relation,
ResourceInstance,
TileModel,
Value as ConceptValue,
)
from arches.app.models.models import ResourceInstance, TileModel

from arches_lingo.const import (
SCHEMES_GRAPH_ID,
Expand All @@ -22,14 +17,10 @@
CONCEPT_NAME_CONTENT_NODE,
CONCEPT_NAME_LANGUAGE_NODE,
CONCEPT_NAME_TYPE_NODE,
HIDDEN_LABEL_VALUE_ID,
LANGUAGE_CONCEPT_ID,
SCHEME_NAME_NODEGROUP,
SCHEME_NAME_CONTENT_NODE,
SCHEME_NAME_LANGUAGE_NODE,
SCHEME_NAME_TYPE_NODE,
PREF_LABEL_VALUE_ID,
ALT_LABEL_VALUE_ID,
)
from arches_lingo.utils.query_expressions import JsonbArrayElements

Expand All @@ -43,8 +34,6 @@ class ConceptBuilder:
def __init__(self):
self.schemes = ResourceInstance.objects.none()

# key=concept valueid (str) val=language code
self.language_concepts: dict[str:str] = {}
# key=scheme resourceid (str) val=set of concept resourceids (str)
self.top_concepts: dict[str : set[str]] = defaultdict(set)
# key=concept resourceid (str) val=set of concept resourceids (str)
Expand All @@ -66,7 +55,6 @@ def __init__(self):
def read_from_cache(self):
from_cache = cache.get_many(
[
"language_concepts",
"top_concepts",
"narrower_concepts",
"schemes",
Expand All @@ -76,7 +64,6 @@ def read_from_cache(self):
]
)
try:
self.language_concepts = from_cache["language_concepts"]
self.top_concepts = from_cache["top_concepts"]
self.narrower_concepts = from_cache["narrower_concepts"]
self.schemes = from_cache["schemes"]
Expand All @@ -87,12 +74,10 @@ def read_from_cache(self):
self.rebuild_cache()

def rebuild_cache(self):
self.language_concepts_map()
self.top_concepts_map()
self.narrower_concepts_map()
self.populate_schemes()

cache.set("language_concepts", self.language_concepts)
cache.set("top_concepts", self.top_concepts)
cache.set("narrower_concepts", self.narrower_concepts)
cache.set("schemes", self.schemes)
Expand All @@ -101,16 +86,6 @@ def rebuild_cache(self):
cache.set("broader_concepts", self.broader_concepts)
cache.set("schemes_by_top_concept", self.schemes_by_top_concept)

@staticmethod
def human_label_type(value_id):
if value_id == PREF_LABEL_VALUE_ID:
return "prefLabel"
if value_id == ALT_LABEL_VALUE_ID:
return "altLabel"
if value_id == HIDDEN_LABEL_VALUE_ID:
return "hidden"
return "unknown"

@staticmethod
def resources_from_tiles(lookup_expression: str):
return CombinedExpression(
Expand All @@ -137,20 +112,6 @@ def labels_subquery(label_nodegroup):
).values("data")
)

def language_concepts_map(self):
language_preflabels = ConceptValue.objects.filter(
Exists(
Relation.objects.filter(
conceptfrom=LANGUAGE_CONCEPT_ID,
conceptto=OuterRef("concept_id"),
relationtype="narrower",
)
),
valuetype="prefLabel",
)
for language_label in language_preflabels:
self.language_concepts[str(language_label.pk)] = language_label.value

def top_concepts_map(self):
top_concept_of_tiles = (
TileModel.objects.filter(nodegroup_id=TOP_CONCEPT_OF_NODE_AND_NODEGROUP)
Expand Down Expand Up @@ -198,15 +159,16 @@ def serialize_scheme(self, scheme: ResourceInstance, *, children=True):
return data

def serialize_scheme_label(self, label_tile: dict):
lang_code = self.language_concepts[label_tile[SCHEME_NAME_LANGUAGE_NODE][0]]
valuetype_id = label_tile[SCHEME_NAME_TYPE_NODE][0]["labels"][0]["value"]
language_id = label_tile[SCHEME_NAME_LANGUAGE_NODE][0]["labels"][0]["value"]
localized_string_objs = label_tile[SCHEME_NAME_CONTENT_NODE].values()
try:
value = next(iter(localized_string_objs))["value"]
except (StopIteration, KeyError):
value = "Unknown"
return {
"valuetype": self.human_label_type(label_tile[SCHEME_NAME_TYPE_NODE]),
"language": lang_code,
"valuetype_id": valuetype_id,
"language_id": language_id,
"value": value,
}

Expand Down Expand Up @@ -256,14 +218,15 @@ def add_broader_concept_recursive(self, working_parent_list, conceptid):
)

def serialize_concept_label(self, label_tile: dict):
lang_code = self.language_concepts[label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]]
valuetype_id = label_tile[CONCEPT_NAME_TYPE_NODE][0]["labels"][0]["value"]
language_id = label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]["labels"][0]["value"]
localized_string_objs = label_tile[CONCEPT_NAME_CONTENT_NODE].values()
try:
value = next(iter(localized_string_objs))["value"]
except (StopIteration, KeyError):
value = "Unknown"
return {
"valuetype": self.human_label_type(label_tile[CONCEPT_NAME_TYPE_NODE]),
"language": lang_code,
"valuetype_id": valuetype_id,
"language_id": language_id,
"value": value,
}
99 changes: 52 additions & 47 deletions tests/tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,17 @@
# these tests can be run from the command line via
# python manage.py test tests.tests --settings="tests.test_settings"

from arches.app.datatypes.datatypes import DataTypeFactory
from arches.app.models.models import (
Concept,
GraphModel,
Node,
NodeGroup,
Relation,
ResourceInstance,
TileModel,
Value,
)

from arches_lingo.const import (
ENGLISH_VALUE_ID,
CONCEPTS_GRAPH_ID,
LANGUAGE_CONCEPT_ID,
SCHEMES_GRAPH_ID,
TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
CLASSIFICATION_STATUS_NODEGROUP,
Expand All @@ -35,52 +31,73 @@
SCHEME_NAME_CONTENT_NODE,
SCHEME_NAME_LANGUAGE_NODE,
SCHEME_NAME_TYPE_NODE,
PREF_LABEL_VALUE_ID,
LANGUAGES_LIST_ID,
LABEL_LIST_ID,
)


def setUpModule():
"""Bootstrap just a few nodes as an alternative to loading the entire package."""
if not GraphModel.objects.filter(pk=SCHEMES_GRAPH_ID).exists():
def localized_string(text, language="en", direction="ltr"):
return {language: {"value": text, "direction": direction}}


class ViewTests(TestCase):
@classmethod
def mock_concept_and_scheme_graphs(cls):
"""Bootstrap just a few nodes as an alternative to loading the entire package."""

GraphModel.objects.create(pk=SCHEMES_GRAPH_ID, isresource=True)
GraphModel.objects.create(pk=CONCEPTS_GRAPH_ID, isresource=True)

for nodegroup_id, node_id, node_name, datatype in [
for nodegroup_id, node_id, node_name, datatype, config in [
(
TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
"top_concept_of",
"concept-list",
"resource-instance-list",
{
"graphs": [{"graphid": SCHEMES_GRAPH_ID, "name": "Scheme"}],
"searchDsl": "",
"searchString": "",
},
),
(
CLASSIFICATION_STATUS_NODEGROUP,
CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID,
"classification_status_ascribed_classification",
"concept-list",
"resource-instance",
{
"graphs": [{"graphid": CONCEPTS_GRAPH_ID, "name": "Concept"}],
"searchDsl": "",
"searchString": "",
},
),
(
SCHEME_NAME_NODEGROUP,
SCHEME_NAME_CONTENT_NODE,
"appellative_status_ascribed_name_content",
"concept-list",
"string",
{"en": ""},
),
(
SCHEME_NAME_NODEGROUP,
SCHEME_NAME_LANGUAGE_NODE,
"appellative_status_ascribed_name_language",
"string",
"reference",
{"controlledList": LANGUAGES_LIST_ID, "multiValue": True},
),
(
CONCEPT_NAME_NODEGROUP,
CONCEPT_NAME_CONTENT_NODE,
"appellative_status_ascribed_name_content",
"concept-list",
"string",
{"en": ""},
),
(
CONCEPT_NAME_NODEGROUP,
CONCEPT_NAME_LANGUAGE_NODE,
"appellative_status_ascribed_name_language",
"string",
"reference",
{"controlledList": LANGUAGES_LIST_ID, "multiValue": True},
),
]:
NodeGroup.objects.update_or_create(
Expand All @@ -93,45 +110,34 @@ def setUpModule():
name=node_name,
istopnode=False,
datatype=datatype,
config=config,
isrequired=datatype == "string",
)

Concept.objects.get_or_create(
conceptid=LANGUAGE_CONCEPT_ID,
nodetype_id="Concept",
)
Value.objects.get_or_create(
concept_id=LANGUAGE_CONCEPT_ID,
valueid=ENGLISH_VALUE_ID,
valuetype_id="prefLabel",
value="en",
)
Relation.objects.get_or_create(
conceptfrom_id=LANGUAGE_CONCEPT_ID,
conceptto_id=LANGUAGE_CONCEPT_ID,
relationtype_id="narrower",
)


def localized_string(text, language="en", direction="ltr"):
return {language: {"value": text, "direction": direction}}


class ViewTests(TestCase):
@classmethod
def setUpTestData(cls):
cls.mock_concept_and_scheme_graphs()
cls.admin = User.objects.get(username="admin")

# Create a scheme with five concepts, each one narrower than the last,
# and each concept after the top concept also narrower than the top.
cls.scheme = ResourceInstance.objects.create(graph_id=SCHEMES_GRAPH_ID)

reference = DataTypeFactory().get_instance("reference")
language_config = {"controlledList": LANGUAGES_LIST_ID}
label_config = {"controlledList": LABEL_LIST_ID}
prefLabel_reference_dt = reference.transform_value_for_tile(
"prefLabel", **label_config
)
en_reference_dt = reference.transform_value_for_tile("en", **language_config)

TileModel.objects.create(
resourceinstance=cls.scheme,
nodegroup_id=SCHEME_NAME_NODEGROUP,
data={
SCHEME_NAME_CONTENT_NODE: localized_string("Test Scheme"),
SCHEME_NAME_TYPE_NODE: PREF_LABEL_VALUE_ID,
SCHEME_NAME_LANGUAGE_NODE: [ENGLISH_VALUE_ID],
SCHEME_NAME_TYPE_NODE: prefLabel_reference_dt,
SCHEME_NAME_LANGUAGE_NODE: en_reference_dt,
},
)

Expand All @@ -150,8 +156,8 @@ def setUpTestData(cls):
nodegroup_id=CONCEPT_NAME_NODEGROUP,
data={
CONCEPT_NAME_CONTENT_NODE: localized_string(f"Concept {i + 1}"),
CONCEPT_NAME_TYPE_NODE: PREF_LABEL_VALUE_ID,
CONCEPT_NAME_LANGUAGE_NODE: [ENGLISH_VALUE_ID],
CONCEPT_NAME_TYPE_NODE: prefLabel_reference_dt,
CONCEPT_NAME_LANGUAGE_NODE: en_reference_dt,
},
)
# Create top concept/narrower tile
Expand Down Expand Up @@ -192,13 +198,12 @@ def setUpTestData(cls):

def test_get_concept_trees(self):
self.client.force_login(self.admin)
with self.assertNumQueries(6):
with self.assertNumQueries(5):
# 1: session
# 2: auth
# 3: select relations (to find languages)
# 4: select broader tiles, subquery for labels
# 5: select top concept tiles, subquery for labels
# 6: select schemes, subquery for labels
# 3: select broader tiles, subquery for labels
# 4: select top concept tiles, subquery for labels
# 5: select schemes, subquery for labels
response = self.client.get(reverse("api-concepts"))

self.assertEqual(response.status_code, 200)
Expand Down

0 comments on commit a67c758

Please sign in to comment.