From 3fe2a15f402a8f9d6e58be3952d0fa06dda88ae5 Mon Sep 17 00:00:00 2001 From: Birger Schacht Date: Wed, 20 Dec 2023 12:46:16 +0100 Subject: [PATCH] refactor(apis_vocabularies)!: drop apis_vocabularies the apis_vocabularies app was removed- it is not being used right now and it will be reindtroduced in #441. BREAKING CHANGE: projects will have to remove `apis_core.apis_vocabularies` from the list of INSTALLED_APPS Closes: #514 --- apis_core/api_routers.py | 36 +--- apis_core/apis_entities/autocomplete3.py | 45 ----- apis_core/apis_entities/forms.py | 34 ---- .../management/commands/serialize_to_cidoc.py | 2 - apis_core/apis_entities/models.py | 6 - .../apis_entities/serializers_generic.py | 8 - .../migrations/0002_auto_20220201_1241.py | 21 -- .../0009_remove_collection_collection_type.py | 17 ++ apis_core/apis_metainfo/models.py | 6 - apis_core/apis_relations/rel_filters.py | 19 -- apis_core/apis_vocabularies/__init__.py | 0 apis_core/apis_vocabularies/admin.py | 75 ------- apis_core/apis_vocabularies/api_renderers.py | 102 ---------- apis_core/apis_vocabularies/apps.py | 6 - .../management/commands/__init__.py | 0 .../management/commands/basic_rel_vocabs.py | 12 -- .../management/commands/import_rel_vocabs.py | 49 ----- .../commands/import_wd_professions.py | 43 ---- .../migrations/0001_initial.py | 191 ------------------ .../apis_vocabularies/migrations/__init__.py | 0 apis_core/apis_vocabularies/models.py | 139 ------------- apis_core/apis_vocabularies/serializers.py | 168 --------------- apis_core/apis_vocabularies/tests.py | 1 - apis_core/apis_vocabularies/urls.py | 17 -- apis_core/apis_vocabularies/views.py | 25 --- apis_core/testrunners.py | 1 - apis_core/urls.py | 5 - apis_core/utils/caching.py | 22 +- apis_core/utils/test_caching.py | 6 - tests/settings_ci.py | 1 - 30 files changed, 22 insertions(+), 1035 deletions(-) create mode 100644 apis_core/apis_metainfo/migrations/0009_remove_collection_collection_type.py delete mode 100644 apis_core/apis_vocabularies/__init__.py delete mode 100644 apis_core/apis_vocabularies/admin.py delete mode 100644 apis_core/apis_vocabularies/api_renderers.py delete mode 100644 apis_core/apis_vocabularies/apps.py delete mode 100644 apis_core/apis_vocabularies/management/commands/__init__.py delete mode 100644 apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py delete mode 100644 apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py delete mode 100644 apis_core/apis_vocabularies/management/commands/import_wd_professions.py delete mode 100644 apis_core/apis_vocabularies/migrations/0001_initial.py delete mode 100644 apis_core/apis_vocabularies/migrations/__init__.py delete mode 100644 apis_core/apis_vocabularies/models.py delete mode 100644 apis_core/apis_vocabularies/serializers.py delete mode 100644 apis_core/apis_vocabularies/tests.py delete mode 100644 apis_core/apis_vocabularies/urls.py delete mode 100644 apis_core/apis_vocabularies/views.py diff --git a/apis_core/api_routers.py b/apis_core/api_routers.py index 7c10aacf3..b5dd0c698 100644 --- a/apis_core/api_routers.py +++ b/apis_core/api_routers.py @@ -82,32 +82,6 @@ def add_uri(self, obj): ) ) - @extend_schema_field(OpenApiTypes.OBJECT) - def add_type(self, obj): - lst_type = ["kind", "type", "collection_type", "relation_type"] - lst_kind = [ - x - for x in obj._meta.fields - if x.name in lst_type and "apis_vocabularies" in str(x.related_model) - ] - if len(lst_kind): - pk_obj = getattr(obj, f"{lst_kind[0].name}_id") - if pk_obj is not None: - obj_type = getattr(obj, str(lst_kind[0].name)) - res = { - "id": pk_obj, - "url": self.context["view"].request.build_absolute_uri( - reverse( - f"apis:apis_api:{lst_kind[0].related_model.__name__.lower()}-detail", - kwargs={"pk": pk_obj}, - ) - ), - "type": obj_type.__class__.__name__, - "label": str(obj_type), - "parent_class": getattr(obj, "parent_class_id", None), - } - return res - class Meta: model = TempEntityClass fields = ["id", "label", "url"] @@ -138,10 +112,6 @@ class Meta(ApisBaseSerializer.Meta): ] -class VocabsBaseSerializer(LabelSerializer, EntitySerializer): - pass - - class RelatedTripleSerializer(ApisBaseSerializer): relation_type = serializers.SerializerMethodField( method_name="add_related_property" @@ -196,9 +166,7 @@ def generic_serializer_creation_factory(): lst_cont = caching.get_all_contenttype_classes() not_allowed_filter_fields = [ "useradded", - "vocab_name", "parent_class", - "vocab", "entity", "autofield", ] @@ -272,7 +240,7 @@ def __init__(self, *args, **kwargs): "ManyToManyField", "ForeignKey", "InheritanceForeignKey", - ] and "apis_vocabularies" not in str(f.related_model): + ]: self.fields[f.name] = ApisBaseSerializer( many=ck_many, read_only=True ) @@ -308,7 +276,7 @@ def __init__(self, *args, **kwargs): elif f.__class__.__name__ in [ "ManyToManyField", "ForeignKey", - ] and "apis_vocabularies" not in str(f.related_model): + ]: self.fields[f.name] = ApisBaseSerializer( many=ck_many, read_only=True ) diff --git a/apis_core/apis_entities/autocomplete3.py b/apis_core/apis_entities/autocomplete3.py index 1dcb8ea91..66b5782da 100644 --- a/apis_core/apis_entities/autocomplete3.py +++ b/apis_core/apis_entities/autocomplete3.py @@ -16,7 +16,6 @@ from django.db.models import Q from apis_core.apis_metainfo.models import Uri, Collection -from apis_core.apis_vocabularies.models import VocabsBaseClass from apis_core.utils import caching from apis_core.utils.caching import get_autocomplete_property_choices from apis_core.utils.settings import get_entity_settings_by_modelname @@ -377,50 +376,6 @@ def get(self, request, *args, **kwargs): ) -# TODO RDF: Check if this should be removed or adapted -class GenericVocabulariesAutocomplete(autocomplete.Select2ListView): - def get(self, request, *args, **kwargs): - page_size = 20 - offset = (int(self.request.GET.get("page", 1)) - 1) * page_size - more = False - vocab = self.kwargs["vocab"] - direct = self.kwargs["direct"] - q = self.q - vocab_model = ContentType.objects.get( - app_label="apis_vocabularies", model=vocab - ).model_class() - if direct == "normal": - if vocab_model.__bases__[0] == VocabsBaseClass: - choices = [ - {"id": x.pk, "text": x.label} - for x in vocab_model.objects.filter(name__icontains=q).order_by( - "parent_class__name", "name" - )[offset : offset + page_size] - ] - else: - choices = [ - {"id": x.pk, "text": x.label} - for x in vocab_model.objects.filter( - Q(name__icontains=q) | Q(name_reverse__icontains=q) - ).order_by("parent_class__name", "name")[ - offset : offset + page_size - ] - ] - elif direct == "reverse": - choices = [ - {"id": x.pk, "text": x.label_reverse} - for x in vocab_model.objects.filter( - Q(name__icontains=q) | Q(name_reverse__icontains=q) - ).order_by("parent_class__name", "name")[offset : offset + page_size] - ] - if len(choices) == page_size: - more = True - return http.HttpResponse( - json.dumps({"results": choices + [], "pagination": {"more": more}}), - content_type="application/json", - ) - - # TODO RDF: Check if this should be removed or adapted class GenericNetworkEntitiesAutocomplete(autocomplete.Select2ListView): def get(self, request, *args, **kwargs): diff --git a/apis_core/apis_entities/forms.py b/apis_core/apis_entities/forms.py index dd397d25f..3b902a4b9 100644 --- a/apis_core/apis_entities/forms.py +++ b/apis_core/apis_entities/forms.py @@ -95,43 +95,9 @@ def __init__(self, *args, **kwargs): "apis_entities", "apis_metainfo", "apis_relations", - "apis_vocabularies", ], model=model_uri, ) - if ( - len(matching_content_type) == 1 - and matching_content_type[0].app_label.lower() - == "apis_vocabularies" - ): - self.fields[f].widget = widget1( - url=reverse( - "apis:apis_vocabularies:generic_vocabularies_autocomplete", - kwargs={"vocab": model_uri, "direct": "normal"}, - ), - attrs=attrs, - ) - if self.instance: - res = [] - if isinstance(self.fields[f], ModelMultipleChoiceField): - try: - for x in getattr(self.instance, f).all(): - res.append((x.pk, x.label)) - except ValueError: - pass - self.fields[f].initial = res - self.fields[f].choices = res - else: - try: - res = getattr(self.instance, f) - if res is not None: - self.fields[f].initial = (res.pk, res.label) - self.fields[f].choices = [ - (res.pk, res.label), - ] - except ValueError: - res = "" - main_fields.append(f) def sort_fields_list(field_names, entity_name): diff --git a/apis_core/apis_entities/management/commands/serialize_to_cidoc.py b/apis_core/apis_entities/management/commands/serialize_to_cidoc.py index 0919c6de9..992b34316 100644 --- a/apis_core/apis_entities/management/commands/serialize_to_cidoc.py +++ b/apis_core/apis_entities/management/commands/serialize_to_cidoc.py @@ -12,8 +12,6 @@ from apis_core.apis_entities.api_renderers import EntityToCIDOC from apis_core.apis_entities.serializers_generic import EntitySerializer -from apis_core.apis_vocabularies.api_renderers import VocabToSkos -from apis_core.apis_vocabularies.serializers import GenericVocabsSerializer from apis_core.utils import caching map_ct = { diff --git a/apis_core/apis_entities/models.py b/apis_core/apis_entities/models.py index 5703d31be..b2475226f 100644 --- a/apis_core/apis_entities/models.py +++ b/apis_core/apis_entities/models.py @@ -336,12 +336,6 @@ def prepare_fields_dict(fields_list, vocabs, vocabs_m2m): res = dict() for f in fields_list: res[f["name"]] = getattr(models, f["field_type"])(**f["attributes"]) - for v in vocabs: - res[v] = models.ForeignKey( - f"apis_vocabularies.{v}", blank=True, null=True, on_delete=models.SET_NULL - ) - for v2 in vocabs_m2m: - res[v2] = models.ManyToManyField(f"apis_vocabularies.{v2}", blank=True) return res diff --git a/apis_core/apis_entities/serializers_generic.py b/apis_core/apis_entities/serializers_generic.py index faba7f547..49ff01a71 100644 --- a/apis_core/apis_entities/serializers_generic.py +++ b/apis_core/apis_entities/serializers_generic.py @@ -140,19 +140,11 @@ def __init__( "FloatField", ]: self.fields[f.name] = getattr(serializers, field_name)() - elif field_name in ["ForeignKey", "ManyToMany"]: - if str(f.related_model.__module__).endswith("apis_vocabularies.models"): - many = False - if f.many_to_many or f.one_to_many: - many = True - self.fields[f.name] = VocabsSerializer(many=many) for f in inst._meta.many_to_many: if f.name.endswith("relationtype_set"): continue elif f.name == "collection": self.fields["collection"] = CollectionSerializer(many=True) - elif str(f.related_model.__module__).endswith("apis_vocabularies.models"): - self.fields[f.name] = VocabsSerializer(many=True) self.fields["entity_type"] = serializers.SerializerMethodField( method_name="add_entity_type" ) diff --git a/apis_core/apis_metainfo/migrations/0002_auto_20220201_1241.py b/apis_core/apis_metainfo/migrations/0002_auto_20220201_1241.py index 1a13c633e..58631ab0f 100644 --- a/apis_core/apis_metainfo/migrations/0002_auto_20220201_1241.py +++ b/apis_core/apis_metainfo/migrations/0002_auto_20220201_1241.py @@ -10,22 +10,11 @@ class Migration(migrations.Migration): dependencies = [ ("apis_metainfo", "0001_initial"), - ("apis_vocabularies", "0001_initial"), ("auth", "0012_alter_user_first_name_max_length"), ("contenttypes", "0002_remove_content_type_name"), ] operations = [ - migrations.AddField( - model_name="text", - name="kind", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="apis_vocabularies.texttype", - ), - ), migrations.AddField( model_name="text", name="source", @@ -46,16 +35,6 @@ class Migration(migrations.Migration): to="contenttypes.contenttype", ), ), - migrations.AddField( - model_name="collection", - name="collection_type", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="apis_vocabularies.collectiontype", - ), - ), migrations.AddField( model_name="collection", name="groups_allowed", diff --git a/apis_core/apis_metainfo/migrations/0009_remove_collection_collection_type.py b/apis_core/apis_metainfo/migrations/0009_remove_collection_collection_type.py new file mode 100644 index 000000000..4a23894b1 --- /dev/null +++ b/apis_core/apis_metainfo/migrations/0009_remove_collection_collection_type.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.8 on 2023-12-20 12:11 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('apis_metainfo', '0008_alter_rootobject_self_contenttype'), + ] + + operations = [ + migrations.RemoveField( + model_name='collection', + name='collection_type', + ), + ] diff --git a/apis_core/apis_metainfo/models.py b/apis_core/apis_metainfo/models.py index bbb10b140..146a921b7 100644 --- a/apis_core/apis_metainfo/models.py +++ b/apis_core/apis_metainfo/models.py @@ -28,7 +28,6 @@ from apis_core.apis_metainfo import signals # from apis_core.apis_entities.serializers_generic import EntitySerializer -# from apis_core.apis_vocabularies.models import CollectionType, LabelType, TextType path_ac_settings = getattr(settings, "APIS_AUTOCOMPLETE_SETTINGS", False) if path_ac_settings: @@ -112,13 +111,8 @@ def duplicate(self): class Collection(models.Model): """Allows to group entities and relation.""" - from apis_core.apis_vocabularies.models import CollectionType - name = models.CharField(max_length=255) description = models.TextField(blank=True) - collection_type = models.ForeignKey( - CollectionType, blank=True, null=True, on_delete=models.SET_NULL - ) groups_allowed = models.ManyToManyField(Group) parent_class = models.ForeignKey( "self", blank=True, null=True, on_delete=models.CASCADE diff --git a/apis_core/apis_relations/rel_filters.py b/apis_core/apis_relations/rel_filters.py index abff57d3c..be9babc6a 100644 --- a/apis_core/apis_relations/rel_filters.py +++ b/apis_core/apis_relations/rel_filters.py @@ -197,25 +197,6 @@ def __init__(self, *args, **kwargs): ), ), ) - elif ( - ContentType.objects.filter( - app_label="apis_vocabularies", model=current_model_name - ).count() - > 0 - ): - self.filters[x] = django_filters.ModelMultipleChoiceFilter( - field_name=x, - queryset=current_qs, - widget=autocomplete.ModelSelect2Multiple( - url=reverse( - "apis:apis_vocabularies:generic_vocabularies_autocomplete", - kwargs={ - "vocab": current_model_name, - "direct": "normal", - }, - ), - ), - ) if type(self.filters[x].field).__name__ == "DateField": self.filters[x] = django_filters.DateFromToRangeFilter( field_name=x, diff --git a/apis_core/apis_vocabularies/__init__.py b/apis_core/apis_vocabularies/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apis_core/apis_vocabularies/admin.py b/apis_core/apis_vocabularies/admin.py deleted file mode 100644 index 9842c17be..000000000 --- a/apis_core/apis_vocabularies/admin.py +++ /dev/null @@ -1,75 +0,0 @@ -from dal import autocomplete -from django.apps import apps -from django.conf import settings -from django.contrib import admin -from django.urls import reverse -from csvexport.actions import csvexport - - -class BaseAdminVocabularies(admin.ModelAdmin): - """Base class used to store the User foreign key in the background when someone adds a vocab.""" - - search_fields = ("name", "parent_class__name") - exclude = ("userAdded",) - actions = [csvexport] - - def get_fields(self, request, obj=None): - lst = super(BaseAdminVocabularies, self).get_fields(request, obj=None) - if not request.user.is_superuser: - lst.remove("status") - return lst - - def get_queryset(self, request): - qs = super(BaseAdminVocabularies, self).get_queryset(request) - if request.user.is_superuser: - return qs.all() - return qs.filter(userAdded__groups__in=request.user.groups.all()).distinct() - - def save_model(self, request, obj, form, change): - if not change: - obj.userAdded = request.user - obj.save() - - def formfield_for_foreignkey(self, db_field, request, **kwargs): - attrs = { - "data-placeholder": "Type to get suggestions", - "data-minimum-input-length": getattr(settings, "APIS_MIN_CHAR", 3), - "data-html": True, - } - c_name = db_field.model.__name__ - qs = super(BaseAdminVocabularies, self).get_queryset(request) - if c_name.endswith("Relation") and db_field.name == "parent_class": - qs = db_field.model - if db_field.name == "parent_class" and request.user.is_superuser: - kwargs["queryset"] = qs.all() - elif db_field.name == "parent_class": - kwargs["queryset"] = qs.filter( - userAdded__groups__in=request.user.groups.all() - ) - kwargs["widget"] = autocomplete.Select2( - url=reverse( - "apis:apis_vocabularies:generic_vocabularies_autocomplete", - kwargs={"vocab": self.model.__name__.lower(), "direct": "normal"}, - ), - attrs=attrs, - ) - - return super(BaseAdminVocabularies, self).formfield_for_foreignkey( - db_field, request, **kwargs - ) - - -class VocabsRelationAdmin(BaseAdminVocabularies): - list_display = ("name", "label") - search_fields = ("name", "parent_class__name") - - -app = apps.get_app_config("apis_vocabularies") - - -for model_name, model in app.models.items(): - - if model_name.endswith("relation"): - admin.site.register(model, VocabsRelationAdmin) - else: - admin.site.register(model, BaseAdminVocabularies) diff --git a/apis_core/apis_vocabularies/api_renderers.py b/apis_core/apis_vocabularies/api_renderers.py deleted file mode 100644 index 3a05577a1..000000000 --- a/apis_core/apis_vocabularies/api_renderers.py +++ /dev/null @@ -1,102 +0,0 @@ -from django.conf import settings -from rdflib import Graph, URIRef, Literal -from rdflib.namespace import SKOS, RDF, DC, RDFS -from rest_framework import renderers -import skosify - -PROJECT_METADATA = getattr(settings, "PROJECT_DEFAULT_MD", {}) - -base_uri_web = getattr(settings, "APIS_BASE_URI", "http://apis.info") -if base_uri_web.endswith("/"): - base_uri_web = base_uri_web[:-1] -lang = getattr(settings, "LANGUAGE_CODE", "de") - - -class VocabToSkos(renderers.BaseRenderer): - - media_type = "text/rdf" - - def render( - self, - data1, - media_type=None, - g=None, - renderer_context=None, - format1=None, - binary=True, - store=None, - ): - if g is None: - g = Graph() - set_skos = getattr(settings, "APIS_SKOSMOS") - base_uri = set_skos.get("url") - if base_uri.endswith("/"): - base_uri = base_uri[:-1] - title = PROJECT_METADATA.get("title", "TITLE") - v_title = set_skos.get("vocabs-name", False) - if not v_title: - v_title = f"{title.lower()}thesaurus/" - uri_1 = f"{base_uri}/{v_title}" - uri = URIRef(f"{uri_1}/{title.title()}Schema") - g.add((uri, RDFS.label, Literal(f"{title}Schema", lang=lang))) - g.add((RDF.Description, RDF.about, uri)) - g.add((uri, RDF.type, SKOS.ConceptScheme)) - g.add( - ( - uri, - DC.description, - Literal(f"{set_skos.get('description', 'DESCRIPTION')}", lang=lang), - ) - ) - g.add( - ( - uri, - DC.license, - URIRef(set_skos.get("license", "https://opensource.org/licenses/MIT")), - ) - ) - g.add((uri, DC.relation, URIRef(base_uri_web))) - - cols = {} - for d in data1: - if not cols.get(d["vocab_name"], False): - vc = URIRef(f"{uri_1}/{d['vocab_name']}") - g.add((vc, RDF.type, SKOS.Collection)) - g.add((vc, SKOS.prefLabel, Literal(d["vocab_name"], lang=lang))) - cols[d["vocab_name"]] = vc - else: - vc = cols.get(d["vocab_name"]) - conc = URIRef(f"{uri_1}/{d['id']}") - g.add((vc, SKOS.member, conc)) - g.add((conc, RDF.type, SKOS.Concept)) - g.add((conc, SKOS.prefLabel, Literal(d["name"], lang=lang))) - if d["name"] != d["label"]: - g.add((conc, SKOS.altLabel, Literal(d["label"], lang=lang))) - rev = d.get("name_reverse", False) - if rev: - g.add((conc, SKOS.altLabel, Literal(f"reverse name: {rev}", lang=lang))) - if d["name_reverse"] != d["label_reverse"]: - g.add( - ( - conc, - SKOS.altLabel, - Literal(f"reverse label: {d['label_reverse']}", lang=lang), - ) - ) - if d.get("userAdded", False): - if (uri, DC.creator, Literal(d["userAdded"], lang=lang)) not in g: - g.add((uri, DC.creator, Literal(d["userAdded"], lang=lang))) - broader = d.get("parent_class", False) - if broader: - g.add((conc, SKOS.broader, URIRef(f"{uri_1}/{broader}"))) - else: - g.add((conc, SKOS.topConceptOf, uri)) - g.add((uri, SKOS.hasTopConcept, conc)) - skosify.infer.skos_related(g) - skosify.infer.skos_topConcept(g) - skosify.infer.skos_hierarchical(g, narrower=True) - skosify.infer.skos_transitive(g, narrower=True) - if format1: - return g.serialize(format1) - elif binary: - return g, store diff --git a/apis_core/apis_vocabularies/apps.py b/apis_core/apis_vocabularies/apps.py deleted file mode 100644 index 84004cd9f..000000000 --- a/apis_core/apis_vocabularies/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class VocabulariesConfig(AppConfig): - default_auto_field = "django.db.models.AutoField" - name = "apis_core.apis_vocabularies" diff --git a/apis_core/apis_vocabularies/management/commands/__init__.py b/apis_core/apis_vocabularies/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py b/apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py deleted file mode 100644 index 6c080b71c..000000000 --- a/apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.apps import apps -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - # Show this when the user types help - help = "creates 'related' vocabs entries for all Relation classes" - - def handle(self, *args, **kwargs): - for x in apps.get_app_config("apis_vocabularies").get_models(): - if x.__name__.endswith("Relation"): - x.objects.get_or_create(name="related", name_reverse="related") diff --git a/apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py b/apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py deleted file mode 100644 index 22b98e127..000000000 --- a/apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py +++ /dev/null @@ -1,49 +0,0 @@ -import pandas as pd -from django.conf import settings -from django.core.management.base import BaseCommand - -cwd = settings.BASE_DIR - - -def create_vocab_item(vocab_class, row, row_key): - """gets or create a vocab entry based on name and name_reverse""" - try: - name_reverse = row[row_key].split("|")[1] - name = row[row_key].split("|")[0] - except IndexError: - name_reverse = row[row_key] - name = row[row_key] - temp_item, _ = vocab_class.objects.get_or_create( - name=name, name_reverse=name_reverse - ) - return temp_item - - -class Command(BaseCommand): - # Show this when the user types help - help = "creates vocab entries (for relations) stored in the passed in Excel-Sheet" - - def add_arguments(self, parser): - parser.add_argument("data", type=str, help="Location of your Excel-Sheet") - - # A command must define handle() - def handle(self, *args, **kwargs): - file = kwargs["data"] - excel_book = pd.read_excel(file, None) - for x in excel_book.keys(): - df = pd.read_excel(file, x) - col_len = len(df.columns) - for i, row in df.iterrows(): - c = 1 - vocab_class = eval(x) - while c <= col_len: - row_key = "ebene_{}".format(c) - if isinstance(row[row_key], str): - temp_item = create_vocab_item(vocab_class, row, row_key) - if c > 1: - parent_key = "ebene_{}".format(c - 1) - parent = create_vocab_item(vocab_class, row, parent_key) - temp_item.parent_class = parent - temp_item.save() - self.stdout.write("{} crated/updated".format(temp_item)) - c += 1 diff --git a/apis_core/apis_vocabularies/management/commands/import_wd_professions.py b/apis_core/apis_vocabularies/management/commands/import_wd_professions.py deleted file mode 100644 index 6cab346e1..000000000 --- a/apis_core/apis_vocabularies/management/commands/import_wd_professions.py +++ /dev/null @@ -1,43 +0,0 @@ -import requests -from django.core.management.base import BaseCommand - -from apis_core.apis_vocabularies.models import ProfessionType - - -class Command(BaseCommand): - # Show this when the user types help - help = "fetches profession entities from wiki data and imports them as apis ProfessionType" - - def add_arguments(self, parser): - parser.add_argument( - "--lang", - default="en", - help="A two digit language code. Defaults to en", - ) - - def handle(self, *args, **options): - url = "https://query.wikidata.org/sparql" - query = """ - SELECT ?item ?itemLabel ?description - WHERE - { - ?item wdt:P31 wd:Q28640. - ?item schema:description ?description FILTER(lang(?description) = "%s") . - SERVICE wikibase:label { bd:serviceParam wikibase:language "%s". } - } - """ - lang = options["lang"] - r = requests.get(url, params={"format": "json", "query": query % (lang, lang)}) - data = r.json() - data = [ - [x["item"]["value"], x["itemLabel"]["value"], x["description"]["value"]] - for x in data["results"]["bindings"] - ] - for x in data: - if x[1].startswith("Q") or x[1].startswith("L"): - pass - else: - label = f"{x[1]} ({x[0].split('/')[-1]})" - item, _ = ProfessionType.objects.get_or_create(name=label) - item.description = x[2] - print(item) diff --git a/apis_core/apis_vocabularies/migrations/0001_initial.py b/apis_core/apis_vocabularies/migrations/0001_initial.py deleted file mode 100644 index 2df0545d3..000000000 --- a/apis_core/apis_vocabularies/migrations/0001_initial.py +++ /dev/null @@ -1,191 +0,0 @@ -# Generated by Django 3.1.14 on 2022-02-01 12:41 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("apis_metainfo", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="VocabNames", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=255)), - ], - ), - migrations.CreateModel( - name="VocabsBaseClass", - fields=[ - ( - "rootobject_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="apis_metainfo.rootobject", - ), - ), - ( - "description", - models.TextField( - blank=True, help_text="Brief description of the used term." - ), - ), - ( - "status", - models.CharField( - choices=[ - ("rej", "rejected"), - ("ac", "accepted"), - ("can", "candidate"), - ("del", "deleted"), - ], - default="can", - max_length=4, - ), - ), - ( - "parent_class", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="apis_vocabularies.vocabsbaseclass", - ), - ), - ( - "userAdded", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "vocab_name", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="apis_vocabularies.vocabnames", - ), - ), - ], - bases=("apis_metainfo.rootobject",), - ), - migrations.CreateModel( - name="CollectionType", - fields=[ - ( - "vocabsbaseclass_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="apis_vocabularies.vocabsbaseclass", - ), - ), - ], - bases=("apis_vocabularies.vocabsbaseclass",), - ), - migrations.CreateModel( - name="LabelType", - fields=[ - ( - "vocabsbaseclass_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="apis_vocabularies.vocabsbaseclass", - ), - ), - ], - bases=("apis_vocabularies.vocabsbaseclass",), - ), - migrations.CreateModel( - name="VocabsUri", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("uri", models.URLField()), - ("domain", models.CharField(blank=True, max_length=255)), - ("rdf_link", models.URLField(blank=True)), - ("loaded", models.BooleanField(default=False)), - ("loaded_time", models.DateTimeField(blank=True, null=True)), - ( - "vocab", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="apis_vocabularies.vocabsbaseclass", - ), - ), - ], - ), - migrations.CreateModel( - name="TextType", - fields=[ - ( - "vocabsbaseclass_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="apis_vocabularies.vocabsbaseclass", - ), - ), - ("entity", models.CharField(max_length=255)), - ( - "lang", - models.CharField( - blank=True, - default="deu", - help_text="The ISO 639-3 (or 2) code for the label's language.", - max_length=3, - null=True, - verbose_name="ISO Code", - ), - ), - ( - "collections", - models.ManyToManyField(blank=True, to="apis_metainfo.Collection"), - ), - ], - bases=("apis_vocabularies.vocabsbaseclass",), - ), - ] diff --git a/apis_core/apis_vocabularies/migrations/__init__.py b/apis_core/apis_vocabularies/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apis_core/apis_vocabularies/models.py b/apis_core/apis_vocabularies/models.py deleted file mode 100644 index c642ec408..000000000 --- a/apis_core/apis_vocabularies/models.py +++ /dev/null @@ -1,139 +0,0 @@ -import re -import unicodedata - -from django.contrib.auth.models import User -from django.db import models -from django.utils.functional import cached_property -from reversion import revisions as reversion -from django.conf import settings -from django.contrib.contenttypes.fields import GenericRelation - -from apis_core.apis_metainfo.models import RootObject - - -@reversion.register() -class VocabNames(models.Model): - """List of Vocabulary names to allow the easy retrieval\ - of Vovcabulary names and classes from the VocabsBaseClass""" - - name = models.CharField(max_length=255) - - def get_vocab_label(self): - return re.sub(r"([A-Z])", r" \1", self.name).strip() - - -# TODO RDF: Remove VocabsBaseClass entirely -@reversion.register() -class VocabsBaseClass(RootObject): - """An abstract base class for other classes which contain so called - 'controlled vocabulary' to describe subtypes of main temporalized - entites""" - - choices_status = ( - ("rej", "rejected"), - ("ac", "accepted"), - ("can", "candidate"), - ("del", "deleted"), - ) - description = models.TextField( - blank=True, help_text="Brief description of the used term." - ) - parent_class = models.ForeignKey( - "self", blank=True, null=True, on_delete=models.CASCADE - ) - status = models.CharField(max_length=4, choices=choices_status, default="can") - userAdded = models.ForeignKey( - User, blank=True, null=True, on_delete=models.SET_NULL - ) - vocab_name = models.ForeignKey( - VocabNames, blank=True, null=True, on_delete=models.SET_NULL - ) - - def __str__(self): - return self.label - - def save(self, *args, **kwargs): - d, created = VocabNames.objects.get_or_create(name=type(self).__name__) - self.vocab_name = d - if self.name != unicodedata.normalize( - "NFC", self.name - ): # secure correct unicode encoding - self.name = unicodedata.normalize("NFC", self.name) - super(VocabsBaseClass, self).save(*args, **kwargs) - return self - - @cached_property - def label(self): - d = self - res = self.name - while d.parent_class: - res = d.parent_class.name + " >> " + res - d = d.parent_class - return res - - -@reversion.register() -class VocabsUri(models.Model): - """Class to store URIs for imported types. URI class from metainfo is not - used in order to keep the vocabularies module/app seperated from the rest of the application. - """ - - uri = models.URLField() - domain = models.CharField(max_length=255, blank=True) - rdf_link = models.URLField(blank=True) - vocab = models.ForeignKey( - VocabsBaseClass, blank=True, null=True, on_delete=models.CASCADE - ) - # loaded: set to True when RDF was loaded and parsed into the data model - loaded = models.BooleanField(default=False) - # loaded_time: Timestamp when file was loaded and parsed - loaded_time = models.DateTimeField(blank=True, null=True) - - def __str__(self): - return self.uri - - -# TODO RDF: Check if this should be removed or adapted -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class WorkType(VocabsBaseClass): -# """Holds controlled vocabularies about work-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class Title(VocabsBaseClass): -# """A person´s (academic) title""" -# abbreviation = models.CharField(max_length=10, blank=True) -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class ProfessionType(VocabsBaseClass): -# """Holds controlled vocabularies about profession-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class PlaceType(VocabsBaseClass): -# """Holds controlled vocabularies about place-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class InstitutionType(VocabsBaseClass): -# """Holds controlled vocabularies about institution-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class EventType(VocabsBaseClass): -# """Holds controlled vocabularies about event-types""" -# pass - - -# TODO RDF: Remove this -@reversion.register(follow=["vocabsbaseclass_ptr"]) -class CollectionType(VocabsBaseClass): - """e.g. reseachCollection, importCollection""" - - pass diff --git a/apis_core/apis_vocabularies/serializers.py b/apis_core/apis_vocabularies/serializers.py deleted file mode 100644 index 6e31413c3..000000000 --- a/apis_core/apis_vocabularies/serializers.py +++ /dev/null @@ -1,168 +0,0 @@ -from django.contrib.auth.models import User -from django.db.models.query import QuerySet -from rest_framework import serializers - -from .models import ( - CollectionType, - VocabsBaseClass, - VocabNames, -) - - -########################################################### -# -# Generic Vocabs Serializer -# -########################################################### - - -class GenericVocabsSerializer(serializers.Serializer): - id = serializers.IntegerField(read_only=True) - userAdded = serializers.SerializerMethodField(method_name="add_user") - parent_class = serializers.PrimaryKeyRelatedField(many=False, read_only=True) - vocab_name = serializers.SerializerMethodField(method_name="add_vocabname") - name = serializers.CharField() - label = serializers.CharField() - - def add_user(self, obj): - print(f"user pk: {obj.userAdded_id}") - try: - u = User.objects.get(pk=obj.userAdded_id) - if u.first_name and u.last_name: - return f"{u.last_name}, {u.first_name}" - else: - return str(u) - except: - return "" - - def add_vocabname(self, obj): - return str(obj.__class__.__name__) - - def __init__(self, *args, **kwargs): - super(GenericVocabsSerializer, self).__init__(*args, **kwargs) - if type(self.instance) == QuerySet: - rt = self.instance[0] - else: - rt = self.instance - if "Relation" in rt.__class__.__name__: - self.fields["name_reverse"] = serializers.CharField() - self.fields["label_reverse"] = serializers.CharField() - - -########################################################### -# -# Meta- Serializers -# -########################################################## - - -class UserAccSerializer(serializers.ModelSerializer): - class Meta: - model = User - fields = ["username", "email"] - - -class VocabsBaseSerializer(serializers.HyperlinkedModelSerializer): - userAdded = serializers.HyperlinkedRelatedField( - view_name="apis:apis_api:user-detail", lookup_field="pk", read_only=True - ) - parent_class = serializers.HyperlinkedRelatedField( - view_name="apis:apis_api:vocabsbaseclass-detail", - lookup_field="pk", - read_only=True, - ) - vocab_name = serializers.HyperlinkedRelatedField( - view_name="apis:apis_api:vocabnames-detail", lookup_field="pk", read_only=True - ) - - -class CollectionTypeSerializer(VocabsBaseSerializer): - url = serializers.HyperlinkedIdentityField( - view_name="apis:apis_api:collectiontype-detail", lookup_field="pk" - ) - - class Meta: - fields = "__all__" - model = CollectionType - - -class VocabsBaseClassSerializer(VocabsBaseSerializer): - url = serializers.HyperlinkedIdentityField( - view_name="apis:apis_api:vocabsbaseclass-detail", lookup_field="pk" - ) - - class Meta: - fields = "__all__" - model = VocabsBaseClass - - -class VocabNamesSerializer(VocabsBaseSerializer): - class Meta: - fields = "__all__" - model = VocabNames - - -############################################################ -# -# Entity Types -# -########################################################### - - -# TODO RDF: Check if this should be removed or adapted -# -# class InstitutionTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:institutiontype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = InstitutionType -# -# -# class ProfessionTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:profession-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = ProfessionType -# -# -# class PlaceTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:placetype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = PlaceType -# -# -# class EventTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:eventtype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = EventType -# -# -# class WorkTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:worktype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = WorkType -# -# diff --git a/apis_core/apis_vocabularies/tests.py b/apis_core/apis_vocabularies/tests.py deleted file mode 100644 index a39b155ac..000000000 --- a/apis_core/apis_vocabularies/tests.py +++ /dev/null @@ -1 +0,0 @@ -# Create your tests here. diff --git a/apis_core/apis_vocabularies/urls.py b/apis_core/apis_vocabularies/urls.py deleted file mode 100644 index 524b89b0b..000000000 --- a/apis_core/apis_vocabularies/urls.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.urls import path, re_path - -from apis_core.apis_entities.autocomplete3 import GenericVocabulariesAutocomplete -from apis_core.apis_vocabularies import views - -app_name = "apis_vocabularies" - -urlpatterns = [ - path("download//", views.dl_vocabs_as_csv, name="dl-vocabs"), - # url(r'^autocomplete/(?P[a-zA-Z0-9-]+)/(?P[a-zA-Z0-9-]+)/$', - re_path( - r"^autocomplete/(?P[a-zA-Z0-9-]+)/$", - # url(r'^autocomplete/$', - GenericVocabulariesAutocomplete.as_view(), - name="generic_vocabularies_autocomplete", - ), -] diff --git a/apis_core/apis_vocabularies/views.py b/apis_core/apis_vocabularies/views.py deleted file mode 100644 index ad1dc9c8a..000000000 --- a/apis_core/apis_vocabularies/views.py +++ /dev/null @@ -1,25 +0,0 @@ -import pandas as pd -from django.contrib.auth.decorators import login_required -from django.contrib.contenttypes.models import ContentType -from django.http import HttpResponse - - -@login_required -def dl_vocabs_as_csv(request, model_name): - kwargs = {"app_label": "apis_vocabularies", "entity": model_name} - model = ContentType.objects.get( - app_label=kwargs.get("app_label"), model=kwargs.get("entity").lower() - ).model_class() - data = [[x.id, x.name, x.parent_class] for x in model.objects.all()] - columns = ["id", "name", "parent_class"] - df = pd.DataFrame(data, columns=columns) - filename = f"{kwargs.get('entity').lower()}.csv" - # Create the HttpResponse object with the appropriate CSV header. - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = f"attachment; filename='{filename}'" - df.to_csv(response, index=False) - - return response - - -# Create your views here. diff --git a/apis_core/testrunners.py b/apis_core/testrunners.py index 1b071c7d0..ebbfb2cc9 100644 --- a/apis_core/testrunners.py +++ b/apis_core/testrunners.py @@ -31,7 +31,6 @@ def setup_databases(self, **kwargs): for x in ct.objects.filter( app_label__in=[ "apis_metainfo", - "apis_vocabularies", "apis_entities", "apis_relations", ] diff --git a/apis_core/urls.py b/apis_core/urls.py index 5b9a2cbf6..ce37eb223 100644 --- a/apis_core/urls.py +++ b/apis_core/urls.py @@ -13,7 +13,6 @@ # NetJsonViewSet, # PlaceGeoJsonViewSet, # ) -# from apis_core.apis_vocabularies.api_views import UserViewSet from apis_core.utils import caching from apis_core.apis_metainfo.viewsets import UriToObjectViewSet from apis_core.core.views import Dumpdata @@ -129,10 +128,6 @@ def build_apis_mock_request(method, path, view, original_request, **kwargs): "relations/", include("apis_core.apis_relations.urls", namespace="apis_relations"), ), - path( - "vocabularies/", - include("apis_core.apis_vocabularies.urls", namespace="apis_vocabularies"), - ), path( "metainfo/", include("apis_core.apis_metainfo.urls", namespace="apis_metainfo"), diff --git a/apis_core/utils/caching.py b/apis_core/utils/caching.py index 8d4a6d568..ad731d829 100644 --- a/apis_core/utils/caching.py +++ b/apis_core/utils/caching.py @@ -10,8 +10,6 @@ _entity_class_names = None # _reification_classes = None # _reification_class_names = None -# _vocabulary_classes = None -# _vocabulary_class_names = None _contenttype_classes = None _contenttype_class_names = None _property_autocomplete_choices = None @@ -21,7 +19,7 @@ def _init_all_ontology_classes(): """ internal function that initiales the lists containing all ontology classes (entities, - reifications, vocabularies) + reifications) :return: None """ @@ -31,7 +29,6 @@ def _init_all_ontology_classes(): from apis_core.apis_entities.models import AbstractEntity, TempEntityClass # from apis_core.apis_relations.models import AbstractReification - # from apis_core.apis_vocabularies.models import AbstractVocabulary global _ontology_classes global _ontology_class_names @@ -39,8 +36,6 @@ def _init_all_ontology_classes(): global _entity_class_names # global _reification_classes # global _reification_class_names - # global _vocabulary_classes - # global _vocabulary_class_names if ( _ontology_classes is not None or _ontology_class_names is not None @@ -48,8 +43,6 @@ def _init_all_ontology_classes(): or _entity_class_names is not None # or _reification_classes is not None # or _reification_class_names is not None - # or _vocabulary_classes is not None - # or _vocabulary_class_names is not None ): raise Exception( "initialization is called but some variables are already initialized." @@ -60,8 +53,6 @@ def _init_all_ontology_classes(): _entity_class_names = [] # _reification_classes = [] # _reification_class_names = [] - # _vocabulary_classes = [] - # _vocabulary_class_names = [] try: from apis_ontology import models as ontology_models @@ -86,16 +77,10 @@ def _init_all_ontology_classes(): # ): # _reification_classes.append(ontology_class) # _reification_class_names.append(ontology_class_name.lower()) - # elif ( - # issubclass(ontology_class, AbstractVocabulary) - # and not ontology_class._meta.abstract - # ): - # _vocabulary_classes.append(ontology_class) - # _vocabulary_class_names.append(ontology_class_name.lower()) - # _ontology_classes = _entity_classes + _reification_classes + _vocabulary_classes + # _ontology_classes = _entity_classes + _reification_classes _ontology_classes = _entity_classes _ontology_class_names = ( - # _entity_class_names + _reification_class_names + _vocabulary_class_names + # _entity_class_names + _reification_class_names _entity_class_names ) @@ -290,7 +275,6 @@ def init_contenttype_classes(): ] apis_modules = [ "apis_core.apis_metainfo.models", - "apis_core.apis_vocabularies.models", "apis_core.apis_entities.models", "apis_core.apis_relations.models", "apis_ontology.models", diff --git a/apis_core/utils/test_caching.py b/apis_core/utils/test_caching.py index f38ee3f9c..04f782c82 100644 --- a/apis_core/utils/test_caching.py +++ b/apis_core/utils/test_caching.py @@ -8,12 +8,6 @@ "RootObject", "Uri", ], - "apis_vocabularies": [ - "CollectionType", - "VocabNames", - "VocabsBaseClass", - "VocabsUri", - ], "apis_entities": ["TempEntityClass"], "apis_relations": ["TempTriple", "Property", "Triple"], "contenttypes": ["ContentType"], diff --git a/tests/settings_ci.py b/tests/settings_ci.py index 8e9464a50..bdf0073d1 100644 --- a/tests/settings_ci.py +++ b/tests/settings_ci.py @@ -14,7 +14,6 @@ "django.contrib.messages", "django.contrib.staticfiles", "apis_core.apis_metainfo", - "apis_core.apis_vocabularies", "apis_core.apis_relations", "apis_core.apis_entities", "reversion",