From 1b5f6a4d40352d35bd112fe3127ea5f4ce3bba8d Mon Sep 17 00:00:00 2001 From: Birger Schacht Date: Fri, 22 Dec 2023 08:57:34 +0100 Subject: [PATCH] refactor(apis_vocabularies)!: remove apis_vocabularies model This removes the `apis_vocabularies` app- it is not being used right now and it will be reindtroduced in #441. The commit removes the app and all references to it from other apps. We drop the whole `apis_vocabularies` Django app - but because there are still references from other apps migrations to this apps migrations and to clean up existing databases, we don't remove the whole module, but still ship the migrations. BREAKING CHANGE: projects will have to remove `apis_core.apis_vocabularies` from the list of INSTALLED_APPS Closes: #514 --- apis_core/api_routers.py | 36 +--- apis_core/apis_entities/autocomplete3.py | 45 ----- apis_core/apis_entities/forms.py | 34 ---- .../management/commands/serialize_to_cidoc.py | 2 - apis_core/apis_entities/models.py | 13 -- .../apis_entities/serializers_generic.py | 8 - apis_core/apis_metainfo/models.py | 2 +- apis_core/apis_relations/rel_filters.py | 19 -- apis_core/apis_vocabularies/__init__.py | 0 apis_core/apis_vocabularies/admin.py | 75 -------- apis_core/apis_vocabularies/api_renderers.py | 102 ----------- apis_core/apis_vocabularies/apps.py | 6 - .../management/commands/__init__.py | 0 .../management/commands/basic_rel_vocabs.py | 12 -- .../management/commands/import_rel_vocabs.py | 49 ----- .../commands/import_wd_professions.py | 43 ----- ...02_remove_texttype_collections_and_more.py | 75 ++++++++ ...0003_delete_vocabnames_delete_vocabsuri.py | 19 ++ .../migrations/0004_delete_collectiontype.py | 16 ++ .../migrations/0005_delete_vocabsbaseclass.py | 16 ++ apis_core/apis_vocabularies/models.py | 139 --------------- apis_core/apis_vocabularies/serializers.py | 168 ------------------ apis_core/apis_vocabularies/tests.py | 1 - apis_core/apis_vocabularies/urls.py | 17 -- apis_core/apis_vocabularies/views.py | 25 --- apis_core/testrunners.py | 3 - apis_core/urls.py | 5 - apis_core/utils/caching.py | 24 +-- apis_core/utils/test_caching.py | 6 - tests/settings_ci.py | 2 +- 30 files changed, 132 insertions(+), 830 deletions(-) delete mode 100644 apis_core/apis_vocabularies/__init__.py delete mode 100644 apis_core/apis_vocabularies/admin.py delete mode 100644 apis_core/apis_vocabularies/api_renderers.py delete mode 100644 apis_core/apis_vocabularies/apps.py delete mode 100644 apis_core/apis_vocabularies/management/commands/__init__.py delete mode 100644 apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py delete mode 100644 apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py delete mode 100644 apis_core/apis_vocabularies/management/commands/import_wd_professions.py create mode 100644 apis_core/apis_vocabularies/migrations/0002_remove_texttype_collections_and_more.py create mode 100644 apis_core/apis_vocabularies/migrations/0003_delete_vocabnames_delete_vocabsuri.py create mode 100644 apis_core/apis_vocabularies/migrations/0004_delete_collectiontype.py create mode 100644 apis_core/apis_vocabularies/migrations/0005_delete_vocabsbaseclass.py delete mode 100644 apis_core/apis_vocabularies/models.py delete mode 100644 apis_core/apis_vocabularies/serializers.py delete mode 100644 apis_core/apis_vocabularies/tests.py delete mode 100644 apis_core/apis_vocabularies/urls.py delete mode 100644 apis_core/apis_vocabularies/views.py diff --git a/apis_core/api_routers.py b/apis_core/api_routers.py index 7c10aacf3..b5dd0c698 100644 --- a/apis_core/api_routers.py +++ b/apis_core/api_routers.py @@ -82,32 +82,6 @@ def add_uri(self, obj): ) ) - @extend_schema_field(OpenApiTypes.OBJECT) - def add_type(self, obj): - lst_type = ["kind", "type", "collection_type", "relation_type"] - lst_kind = [ - x - for x in obj._meta.fields - if x.name in lst_type and "apis_vocabularies" in str(x.related_model) - ] - if len(lst_kind): - pk_obj = getattr(obj, f"{lst_kind[0].name}_id") - if pk_obj is not None: - obj_type = getattr(obj, str(lst_kind[0].name)) - res = { - "id": pk_obj, - "url": self.context["view"].request.build_absolute_uri( - reverse( - f"apis:apis_api:{lst_kind[0].related_model.__name__.lower()}-detail", - kwargs={"pk": pk_obj}, - ) - ), - "type": obj_type.__class__.__name__, - "label": str(obj_type), - "parent_class": getattr(obj, "parent_class_id", None), - } - return res - class Meta: model = TempEntityClass fields = ["id", "label", "url"] @@ -138,10 +112,6 @@ class Meta(ApisBaseSerializer.Meta): ] -class VocabsBaseSerializer(LabelSerializer, EntitySerializer): - pass - - class RelatedTripleSerializer(ApisBaseSerializer): relation_type = serializers.SerializerMethodField( method_name="add_related_property" @@ -196,9 +166,7 @@ def generic_serializer_creation_factory(): lst_cont = caching.get_all_contenttype_classes() not_allowed_filter_fields = [ "useradded", - "vocab_name", "parent_class", - "vocab", "entity", "autofield", ] @@ -272,7 +240,7 @@ def __init__(self, *args, **kwargs): "ManyToManyField", "ForeignKey", "InheritanceForeignKey", - ] and "apis_vocabularies" not in str(f.related_model): + ]: self.fields[f.name] = ApisBaseSerializer( many=ck_many, read_only=True ) @@ -308,7 +276,7 @@ def __init__(self, *args, **kwargs): elif f.__class__.__name__ in [ "ManyToManyField", "ForeignKey", - ] and "apis_vocabularies" not in str(f.related_model): + ]: self.fields[f.name] = ApisBaseSerializer( many=ck_many, read_only=True ) diff --git a/apis_core/apis_entities/autocomplete3.py b/apis_core/apis_entities/autocomplete3.py index 1dcb8ea91..66b5782da 100644 --- a/apis_core/apis_entities/autocomplete3.py +++ b/apis_core/apis_entities/autocomplete3.py @@ -16,7 +16,6 @@ from django.db.models import Q from apis_core.apis_metainfo.models import Uri, Collection -from apis_core.apis_vocabularies.models import VocabsBaseClass from apis_core.utils import caching from apis_core.utils.caching import get_autocomplete_property_choices from apis_core.utils.settings import get_entity_settings_by_modelname @@ -377,50 +376,6 @@ def get(self, request, *args, **kwargs): ) -# TODO RDF: Check if this should be removed or adapted -class GenericVocabulariesAutocomplete(autocomplete.Select2ListView): - def get(self, request, *args, **kwargs): - page_size = 20 - offset = (int(self.request.GET.get("page", 1)) - 1) * page_size - more = False - vocab = self.kwargs["vocab"] - direct = self.kwargs["direct"] - q = self.q - vocab_model = ContentType.objects.get( - app_label="apis_vocabularies", model=vocab - ).model_class() - if direct == "normal": - if vocab_model.__bases__[0] == VocabsBaseClass: - choices = [ - {"id": x.pk, "text": x.label} - for x in vocab_model.objects.filter(name__icontains=q).order_by( - "parent_class__name", "name" - )[offset : offset + page_size] - ] - else: - choices = [ - {"id": x.pk, "text": x.label} - for x in vocab_model.objects.filter( - Q(name__icontains=q) | Q(name_reverse__icontains=q) - ).order_by("parent_class__name", "name")[ - offset : offset + page_size - ] - ] - elif direct == "reverse": - choices = [ - {"id": x.pk, "text": x.label_reverse} - for x in vocab_model.objects.filter( - Q(name__icontains=q) | Q(name_reverse__icontains=q) - ).order_by("parent_class__name", "name")[offset : offset + page_size] - ] - if len(choices) == page_size: - more = True - return http.HttpResponse( - json.dumps({"results": choices + [], "pagination": {"more": more}}), - content_type="application/json", - ) - - # TODO RDF: Check if this should be removed or adapted class GenericNetworkEntitiesAutocomplete(autocomplete.Select2ListView): def get(self, request, *args, **kwargs): diff --git a/apis_core/apis_entities/forms.py b/apis_core/apis_entities/forms.py index dd397d25f..3b902a4b9 100644 --- a/apis_core/apis_entities/forms.py +++ b/apis_core/apis_entities/forms.py @@ -95,43 +95,9 @@ def __init__(self, *args, **kwargs): "apis_entities", "apis_metainfo", "apis_relations", - "apis_vocabularies", ], model=model_uri, ) - if ( - len(matching_content_type) == 1 - and matching_content_type[0].app_label.lower() - == "apis_vocabularies" - ): - self.fields[f].widget = widget1( - url=reverse( - "apis:apis_vocabularies:generic_vocabularies_autocomplete", - kwargs={"vocab": model_uri, "direct": "normal"}, - ), - attrs=attrs, - ) - if self.instance: - res = [] - if isinstance(self.fields[f], ModelMultipleChoiceField): - try: - for x in getattr(self.instance, f).all(): - res.append((x.pk, x.label)) - except ValueError: - pass - self.fields[f].initial = res - self.fields[f].choices = res - else: - try: - res = getattr(self.instance, f) - if res is not None: - self.fields[f].initial = (res.pk, res.label) - self.fields[f].choices = [ - (res.pk, res.label), - ] - except ValueError: - res = "" - main_fields.append(f) def sort_fields_list(field_names, entity_name): diff --git a/apis_core/apis_entities/management/commands/serialize_to_cidoc.py b/apis_core/apis_entities/management/commands/serialize_to_cidoc.py index 0919c6de9..992b34316 100644 --- a/apis_core/apis_entities/management/commands/serialize_to_cidoc.py +++ b/apis_core/apis_entities/management/commands/serialize_to_cidoc.py @@ -12,8 +12,6 @@ from apis_core.apis_entities.api_renderers import EntityToCIDOC from apis_core.apis_entities.serializers_generic import EntitySerializer -from apis_core.apis_vocabularies.api_renderers import VocabToSkos -from apis_core.apis_vocabularies.serializers import GenericVocabsSerializer from apis_core.utils import caching map_ct = { diff --git a/apis_core/apis_entities/models.py b/apis_core/apis_entities/models.py index 5703d31be..444e7d861 100644 --- a/apis_core/apis_entities/models.py +++ b/apis_core/apis_entities/models.py @@ -332,19 +332,6 @@ def save(self, parse_dates=True, *args, **kwargs): return self -def prepare_fields_dict(fields_list, vocabs, vocabs_m2m): - res = dict() - for f in fields_list: - res[f["name"]] = getattr(models, f["field_type"])(**f["attributes"]) - for v in vocabs: - res[v] = models.ForeignKey( - f"apis_vocabularies.{v}", blank=True, null=True, on_delete=models.SET_NULL - ) - for v2 in vocabs_m2m: - res[v2] = models.ManyToManyField(f"apis_vocabularies.{v2}", blank=True) - return res - - @receiver(post_save, dispatch_uid="create_default_uri") def create_default_uri(sender, instance, raw, **kwargs): # with django reversion, browsing deleted entries in the admin interface diff --git a/apis_core/apis_entities/serializers_generic.py b/apis_core/apis_entities/serializers_generic.py index faba7f547..49ff01a71 100644 --- a/apis_core/apis_entities/serializers_generic.py +++ b/apis_core/apis_entities/serializers_generic.py @@ -140,19 +140,11 @@ def __init__( "FloatField", ]: self.fields[f.name] = getattr(serializers, field_name)() - elif field_name in ["ForeignKey", "ManyToMany"]: - if str(f.related_model.__module__).endswith("apis_vocabularies.models"): - many = False - if f.many_to_many or f.one_to_many: - many = True - self.fields[f.name] = VocabsSerializer(many=many) for f in inst._meta.many_to_many: if f.name.endswith("relationtype_set"): continue elif f.name == "collection": self.fields["collection"] = CollectionSerializer(many=True) - elif str(f.related_model.__module__).endswith("apis_vocabularies.models"): - self.fields[f.name] = VocabsSerializer(many=True) self.fields["entity_type"] = serializers.SerializerMethodField( method_name="add_entity_type" ) diff --git a/apis_core/apis_metainfo/models.py b/apis_core/apis_metainfo/models.py index 7bd9fff09..146a921b7 100644 --- a/apis_core/apis_metainfo/models.py +++ b/apis_core/apis_metainfo/models.py @@ -28,7 +28,6 @@ from apis_core.apis_metainfo import signals # from apis_core.apis_entities.serializers_generic import EntitySerializer -# from apis_core.apis_vocabularies.models import CollectionType, LabelType, TextType path_ac_settings = getattr(settings, "APIS_AUTOCOMPLETE_SETTINGS", False) if path_ac_settings: @@ -111,6 +110,7 @@ def duplicate(self): @reversion.register() class Collection(models.Model): """Allows to group entities and relation.""" + name = models.CharField(max_length=255) description = models.TextField(blank=True) groups_allowed = models.ManyToManyField(Group) diff --git a/apis_core/apis_relations/rel_filters.py b/apis_core/apis_relations/rel_filters.py index abff57d3c..be9babc6a 100644 --- a/apis_core/apis_relations/rel_filters.py +++ b/apis_core/apis_relations/rel_filters.py @@ -197,25 +197,6 @@ def __init__(self, *args, **kwargs): ), ), ) - elif ( - ContentType.objects.filter( - app_label="apis_vocabularies", model=current_model_name - ).count() - > 0 - ): - self.filters[x] = django_filters.ModelMultipleChoiceFilter( - field_name=x, - queryset=current_qs, - widget=autocomplete.ModelSelect2Multiple( - url=reverse( - "apis:apis_vocabularies:generic_vocabularies_autocomplete", - kwargs={ - "vocab": current_model_name, - "direct": "normal", - }, - ), - ), - ) if type(self.filters[x].field).__name__ == "DateField": self.filters[x] = django_filters.DateFromToRangeFilter( field_name=x, diff --git a/apis_core/apis_vocabularies/__init__.py b/apis_core/apis_vocabularies/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apis_core/apis_vocabularies/admin.py b/apis_core/apis_vocabularies/admin.py deleted file mode 100644 index 9842c17be..000000000 --- a/apis_core/apis_vocabularies/admin.py +++ /dev/null @@ -1,75 +0,0 @@ -from dal import autocomplete -from django.apps import apps -from django.conf import settings -from django.contrib import admin -from django.urls import reverse -from csvexport.actions import csvexport - - -class BaseAdminVocabularies(admin.ModelAdmin): - """Base class used to store the User foreign key in the background when someone adds a vocab.""" - - search_fields = ("name", "parent_class__name") - exclude = ("userAdded",) - actions = [csvexport] - - def get_fields(self, request, obj=None): - lst = super(BaseAdminVocabularies, self).get_fields(request, obj=None) - if not request.user.is_superuser: - lst.remove("status") - return lst - - def get_queryset(self, request): - qs = super(BaseAdminVocabularies, self).get_queryset(request) - if request.user.is_superuser: - return qs.all() - return qs.filter(userAdded__groups__in=request.user.groups.all()).distinct() - - def save_model(self, request, obj, form, change): - if not change: - obj.userAdded = request.user - obj.save() - - def formfield_for_foreignkey(self, db_field, request, **kwargs): - attrs = { - "data-placeholder": "Type to get suggestions", - "data-minimum-input-length": getattr(settings, "APIS_MIN_CHAR", 3), - "data-html": True, - } - c_name = db_field.model.__name__ - qs = super(BaseAdminVocabularies, self).get_queryset(request) - if c_name.endswith("Relation") and db_field.name == "parent_class": - qs = db_field.model - if db_field.name == "parent_class" and request.user.is_superuser: - kwargs["queryset"] = qs.all() - elif db_field.name == "parent_class": - kwargs["queryset"] = qs.filter( - userAdded__groups__in=request.user.groups.all() - ) - kwargs["widget"] = autocomplete.Select2( - url=reverse( - "apis:apis_vocabularies:generic_vocabularies_autocomplete", - kwargs={"vocab": self.model.__name__.lower(), "direct": "normal"}, - ), - attrs=attrs, - ) - - return super(BaseAdminVocabularies, self).formfield_for_foreignkey( - db_field, request, **kwargs - ) - - -class VocabsRelationAdmin(BaseAdminVocabularies): - list_display = ("name", "label") - search_fields = ("name", "parent_class__name") - - -app = apps.get_app_config("apis_vocabularies") - - -for model_name, model in app.models.items(): - - if model_name.endswith("relation"): - admin.site.register(model, VocabsRelationAdmin) - else: - admin.site.register(model, BaseAdminVocabularies) diff --git a/apis_core/apis_vocabularies/api_renderers.py b/apis_core/apis_vocabularies/api_renderers.py deleted file mode 100644 index 3a05577a1..000000000 --- a/apis_core/apis_vocabularies/api_renderers.py +++ /dev/null @@ -1,102 +0,0 @@ -from django.conf import settings -from rdflib import Graph, URIRef, Literal -from rdflib.namespace import SKOS, RDF, DC, RDFS -from rest_framework import renderers -import skosify - -PROJECT_METADATA = getattr(settings, "PROJECT_DEFAULT_MD", {}) - -base_uri_web = getattr(settings, "APIS_BASE_URI", "http://apis.info") -if base_uri_web.endswith("/"): - base_uri_web = base_uri_web[:-1] -lang = getattr(settings, "LANGUAGE_CODE", "de") - - -class VocabToSkos(renderers.BaseRenderer): - - media_type = "text/rdf" - - def render( - self, - data1, - media_type=None, - g=None, - renderer_context=None, - format1=None, - binary=True, - store=None, - ): - if g is None: - g = Graph() - set_skos = getattr(settings, "APIS_SKOSMOS") - base_uri = set_skos.get("url") - if base_uri.endswith("/"): - base_uri = base_uri[:-1] - title = PROJECT_METADATA.get("title", "TITLE") - v_title = set_skos.get("vocabs-name", False) - if not v_title: - v_title = f"{title.lower()}thesaurus/" - uri_1 = f"{base_uri}/{v_title}" - uri = URIRef(f"{uri_1}/{title.title()}Schema") - g.add((uri, RDFS.label, Literal(f"{title}Schema", lang=lang))) - g.add((RDF.Description, RDF.about, uri)) - g.add((uri, RDF.type, SKOS.ConceptScheme)) - g.add( - ( - uri, - DC.description, - Literal(f"{set_skos.get('description', 'DESCRIPTION')}", lang=lang), - ) - ) - g.add( - ( - uri, - DC.license, - URIRef(set_skos.get("license", "https://opensource.org/licenses/MIT")), - ) - ) - g.add((uri, DC.relation, URIRef(base_uri_web))) - - cols = {} - for d in data1: - if not cols.get(d["vocab_name"], False): - vc = URIRef(f"{uri_1}/{d['vocab_name']}") - g.add((vc, RDF.type, SKOS.Collection)) - g.add((vc, SKOS.prefLabel, Literal(d["vocab_name"], lang=lang))) - cols[d["vocab_name"]] = vc - else: - vc = cols.get(d["vocab_name"]) - conc = URIRef(f"{uri_1}/{d['id']}") - g.add((vc, SKOS.member, conc)) - g.add((conc, RDF.type, SKOS.Concept)) - g.add((conc, SKOS.prefLabel, Literal(d["name"], lang=lang))) - if d["name"] != d["label"]: - g.add((conc, SKOS.altLabel, Literal(d["label"], lang=lang))) - rev = d.get("name_reverse", False) - if rev: - g.add((conc, SKOS.altLabel, Literal(f"reverse name: {rev}", lang=lang))) - if d["name_reverse"] != d["label_reverse"]: - g.add( - ( - conc, - SKOS.altLabel, - Literal(f"reverse label: {d['label_reverse']}", lang=lang), - ) - ) - if d.get("userAdded", False): - if (uri, DC.creator, Literal(d["userAdded"], lang=lang)) not in g: - g.add((uri, DC.creator, Literal(d["userAdded"], lang=lang))) - broader = d.get("parent_class", False) - if broader: - g.add((conc, SKOS.broader, URIRef(f"{uri_1}/{broader}"))) - else: - g.add((conc, SKOS.topConceptOf, uri)) - g.add((uri, SKOS.hasTopConcept, conc)) - skosify.infer.skos_related(g) - skosify.infer.skos_topConcept(g) - skosify.infer.skos_hierarchical(g, narrower=True) - skosify.infer.skos_transitive(g, narrower=True) - if format1: - return g.serialize(format1) - elif binary: - return g, store diff --git a/apis_core/apis_vocabularies/apps.py b/apis_core/apis_vocabularies/apps.py deleted file mode 100644 index 84004cd9f..000000000 --- a/apis_core/apis_vocabularies/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class VocabulariesConfig(AppConfig): - default_auto_field = "django.db.models.AutoField" - name = "apis_core.apis_vocabularies" diff --git a/apis_core/apis_vocabularies/management/commands/__init__.py b/apis_core/apis_vocabularies/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py b/apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py deleted file mode 100644 index 6c080b71c..000000000 --- a/apis_core/apis_vocabularies/management/commands/basic_rel_vocabs.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.apps import apps -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - # Show this when the user types help - help = "creates 'related' vocabs entries for all Relation classes" - - def handle(self, *args, **kwargs): - for x in apps.get_app_config("apis_vocabularies").get_models(): - if x.__name__.endswith("Relation"): - x.objects.get_or_create(name="related", name_reverse="related") diff --git a/apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py b/apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py deleted file mode 100644 index 22b98e127..000000000 --- a/apis_core/apis_vocabularies/management/commands/import_rel_vocabs.py +++ /dev/null @@ -1,49 +0,0 @@ -import pandas as pd -from django.conf import settings -from django.core.management.base import BaseCommand - -cwd = settings.BASE_DIR - - -def create_vocab_item(vocab_class, row, row_key): - """gets or create a vocab entry based on name and name_reverse""" - try: - name_reverse = row[row_key].split("|")[1] - name = row[row_key].split("|")[0] - except IndexError: - name_reverse = row[row_key] - name = row[row_key] - temp_item, _ = vocab_class.objects.get_or_create( - name=name, name_reverse=name_reverse - ) - return temp_item - - -class Command(BaseCommand): - # Show this when the user types help - help = "creates vocab entries (for relations) stored in the passed in Excel-Sheet" - - def add_arguments(self, parser): - parser.add_argument("data", type=str, help="Location of your Excel-Sheet") - - # A command must define handle() - def handle(self, *args, **kwargs): - file = kwargs["data"] - excel_book = pd.read_excel(file, None) - for x in excel_book.keys(): - df = pd.read_excel(file, x) - col_len = len(df.columns) - for i, row in df.iterrows(): - c = 1 - vocab_class = eval(x) - while c <= col_len: - row_key = "ebene_{}".format(c) - if isinstance(row[row_key], str): - temp_item = create_vocab_item(vocab_class, row, row_key) - if c > 1: - parent_key = "ebene_{}".format(c - 1) - parent = create_vocab_item(vocab_class, row, parent_key) - temp_item.parent_class = parent - temp_item.save() - self.stdout.write("{} crated/updated".format(temp_item)) - c += 1 diff --git a/apis_core/apis_vocabularies/management/commands/import_wd_professions.py b/apis_core/apis_vocabularies/management/commands/import_wd_professions.py deleted file mode 100644 index 6cab346e1..000000000 --- a/apis_core/apis_vocabularies/management/commands/import_wd_professions.py +++ /dev/null @@ -1,43 +0,0 @@ -import requests -from django.core.management.base import BaseCommand - -from apis_core.apis_vocabularies.models import ProfessionType - - -class Command(BaseCommand): - # Show this when the user types help - help = "fetches profession entities from wiki data and imports them as apis ProfessionType" - - def add_arguments(self, parser): - parser.add_argument( - "--lang", - default="en", - help="A two digit language code. Defaults to en", - ) - - def handle(self, *args, **options): - url = "https://query.wikidata.org/sparql" - query = """ - SELECT ?item ?itemLabel ?description - WHERE - { - ?item wdt:P31 wd:Q28640. - ?item schema:description ?description FILTER(lang(?description) = "%s") . - SERVICE wikibase:label { bd:serviceParam wikibase:language "%s". } - } - """ - lang = options["lang"] - r = requests.get(url, params={"format": "json", "query": query % (lang, lang)}) - data = r.json() - data = [ - [x["item"]["value"], x["itemLabel"]["value"], x["description"]["value"]] - for x in data["results"]["bindings"] - ] - for x in data: - if x[1].startswith("Q") or x[1].startswith("L"): - pass - else: - label = f"{x[1]} ({x[0].split('/')[-1]})" - item, _ = ProfessionType.objects.get_or_create(name=label) - item.description = x[2] - print(item) diff --git a/apis_core/apis_vocabularies/migrations/0002_remove_texttype_collections_and_more.py b/apis_core/apis_vocabularies/migrations/0002_remove_texttype_collections_and_more.py new file mode 100644 index 000000000..4a2a6ce1f --- /dev/null +++ b/apis_core/apis_vocabularies/migrations/0002_remove_texttype_collections_and_more.py @@ -0,0 +1,75 @@ +# Generated by Django 4.2.8 on 2023-12-22 07:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("apis_vocabularies", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="texttype", + name="collections", + ), + migrations.RemoveField( + model_name="texttype", + name="vocabsbaseclass_ptr", + ), + migrations.RemoveField( + model_name="vocabnames", + name="name", + ), + migrations.RemoveField( + model_name="vocabsbaseclass", + name="description", + ), + migrations.RemoveField( + model_name="vocabsbaseclass", + name="parent_class", + ), + migrations.RemoveField( + model_name="vocabsbaseclass", + name="status", + ), + migrations.RemoveField( + model_name="vocabsbaseclass", + name="userAdded", + ), + migrations.RemoveField( + model_name="vocabsbaseclass", + name="vocab_name", + ), + migrations.RemoveField( + model_name="vocabsuri", + name="domain", + ), + migrations.RemoveField( + model_name="vocabsuri", + name="loaded", + ), + migrations.RemoveField( + model_name="vocabsuri", + name="loaded_time", + ), + migrations.RemoveField( + model_name="vocabsuri", + name="rdf_link", + ), + migrations.RemoveField( + model_name="vocabsuri", + name="uri", + ), + migrations.RemoveField( + model_name="vocabsuri", + name="vocab", + ), + migrations.DeleteModel( + name="LabelType", + ), + migrations.DeleteModel( + name="TextType", + ), + ] diff --git a/apis_core/apis_vocabularies/migrations/0003_delete_vocabnames_delete_vocabsuri.py b/apis_core/apis_vocabularies/migrations/0003_delete_vocabnames_delete_vocabsuri.py new file mode 100644 index 000000000..b368b4012 --- /dev/null +++ b/apis_core/apis_vocabularies/migrations/0003_delete_vocabnames_delete_vocabsuri.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.8 on 2023-12-22 07:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("apis_vocabularies", "0002_remove_texttype_collections_and_more"), + ] + + operations = [ + migrations.DeleteModel( + name="VocabNames", + ), + migrations.DeleteModel( + name="VocabsUri", + ), + ] diff --git a/apis_core/apis_vocabularies/migrations/0004_delete_collectiontype.py b/apis_core/apis_vocabularies/migrations/0004_delete_collectiontype.py new file mode 100644 index 000000000..1f24ebfcb --- /dev/null +++ b/apis_core/apis_vocabularies/migrations/0004_delete_collectiontype.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.8 on 2023-12-22 07:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("apis_vocabularies", "0003_delete_vocabnames_delete_vocabsuri"), + ] + + operations = [ + migrations.DeleteModel( + name="CollectionType", + ), + ] diff --git a/apis_core/apis_vocabularies/migrations/0005_delete_vocabsbaseclass.py b/apis_core/apis_vocabularies/migrations/0005_delete_vocabsbaseclass.py new file mode 100644 index 000000000..cb9aa5006 --- /dev/null +++ b/apis_core/apis_vocabularies/migrations/0005_delete_vocabsbaseclass.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.8 on 2023-12-22 07:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("apis_vocabularies", "0004_delete_collectiontype"), + ] + + operations = [ + migrations.DeleteModel( + name="VocabsBaseClass", + ), + ] diff --git a/apis_core/apis_vocabularies/models.py b/apis_core/apis_vocabularies/models.py deleted file mode 100644 index c642ec408..000000000 --- a/apis_core/apis_vocabularies/models.py +++ /dev/null @@ -1,139 +0,0 @@ -import re -import unicodedata - -from django.contrib.auth.models import User -from django.db import models -from django.utils.functional import cached_property -from reversion import revisions as reversion -from django.conf import settings -from django.contrib.contenttypes.fields import GenericRelation - -from apis_core.apis_metainfo.models import RootObject - - -@reversion.register() -class VocabNames(models.Model): - """List of Vocabulary names to allow the easy retrieval\ - of Vovcabulary names and classes from the VocabsBaseClass""" - - name = models.CharField(max_length=255) - - def get_vocab_label(self): - return re.sub(r"([A-Z])", r" \1", self.name).strip() - - -# TODO RDF: Remove VocabsBaseClass entirely -@reversion.register() -class VocabsBaseClass(RootObject): - """An abstract base class for other classes which contain so called - 'controlled vocabulary' to describe subtypes of main temporalized - entites""" - - choices_status = ( - ("rej", "rejected"), - ("ac", "accepted"), - ("can", "candidate"), - ("del", "deleted"), - ) - description = models.TextField( - blank=True, help_text="Brief description of the used term." - ) - parent_class = models.ForeignKey( - "self", blank=True, null=True, on_delete=models.CASCADE - ) - status = models.CharField(max_length=4, choices=choices_status, default="can") - userAdded = models.ForeignKey( - User, blank=True, null=True, on_delete=models.SET_NULL - ) - vocab_name = models.ForeignKey( - VocabNames, blank=True, null=True, on_delete=models.SET_NULL - ) - - def __str__(self): - return self.label - - def save(self, *args, **kwargs): - d, created = VocabNames.objects.get_or_create(name=type(self).__name__) - self.vocab_name = d - if self.name != unicodedata.normalize( - "NFC", self.name - ): # secure correct unicode encoding - self.name = unicodedata.normalize("NFC", self.name) - super(VocabsBaseClass, self).save(*args, **kwargs) - return self - - @cached_property - def label(self): - d = self - res = self.name - while d.parent_class: - res = d.parent_class.name + " >> " + res - d = d.parent_class - return res - - -@reversion.register() -class VocabsUri(models.Model): - """Class to store URIs for imported types. URI class from metainfo is not - used in order to keep the vocabularies module/app seperated from the rest of the application. - """ - - uri = models.URLField() - domain = models.CharField(max_length=255, blank=True) - rdf_link = models.URLField(blank=True) - vocab = models.ForeignKey( - VocabsBaseClass, blank=True, null=True, on_delete=models.CASCADE - ) - # loaded: set to True when RDF was loaded and parsed into the data model - loaded = models.BooleanField(default=False) - # loaded_time: Timestamp when file was loaded and parsed - loaded_time = models.DateTimeField(blank=True, null=True) - - def __str__(self): - return self.uri - - -# TODO RDF: Check if this should be removed or adapted -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class WorkType(VocabsBaseClass): -# """Holds controlled vocabularies about work-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class Title(VocabsBaseClass): -# """A person´s (academic) title""" -# abbreviation = models.CharField(max_length=10, blank=True) -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class ProfessionType(VocabsBaseClass): -# """Holds controlled vocabularies about profession-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class PlaceType(VocabsBaseClass): -# """Holds controlled vocabularies about place-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class InstitutionType(VocabsBaseClass): -# """Holds controlled vocabularies about institution-types""" -# pass -# -# -# @reversion.register(follow=['vocabsbaseclass_ptr']) -# class EventType(VocabsBaseClass): -# """Holds controlled vocabularies about event-types""" -# pass - - -# TODO RDF: Remove this -@reversion.register(follow=["vocabsbaseclass_ptr"]) -class CollectionType(VocabsBaseClass): - """e.g. reseachCollection, importCollection""" - - pass diff --git a/apis_core/apis_vocabularies/serializers.py b/apis_core/apis_vocabularies/serializers.py deleted file mode 100644 index 6e31413c3..000000000 --- a/apis_core/apis_vocabularies/serializers.py +++ /dev/null @@ -1,168 +0,0 @@ -from django.contrib.auth.models import User -from django.db.models.query import QuerySet -from rest_framework import serializers - -from .models import ( - CollectionType, - VocabsBaseClass, - VocabNames, -) - - -########################################################### -# -# Generic Vocabs Serializer -# -########################################################### - - -class GenericVocabsSerializer(serializers.Serializer): - id = serializers.IntegerField(read_only=True) - userAdded = serializers.SerializerMethodField(method_name="add_user") - parent_class = serializers.PrimaryKeyRelatedField(many=False, read_only=True) - vocab_name = serializers.SerializerMethodField(method_name="add_vocabname") - name = serializers.CharField() - label = serializers.CharField() - - def add_user(self, obj): - print(f"user pk: {obj.userAdded_id}") - try: - u = User.objects.get(pk=obj.userAdded_id) - if u.first_name and u.last_name: - return f"{u.last_name}, {u.first_name}" - else: - return str(u) - except: - return "" - - def add_vocabname(self, obj): - return str(obj.__class__.__name__) - - def __init__(self, *args, **kwargs): - super(GenericVocabsSerializer, self).__init__(*args, **kwargs) - if type(self.instance) == QuerySet: - rt = self.instance[0] - else: - rt = self.instance - if "Relation" in rt.__class__.__name__: - self.fields["name_reverse"] = serializers.CharField() - self.fields["label_reverse"] = serializers.CharField() - - -########################################################### -# -# Meta- Serializers -# -########################################################## - - -class UserAccSerializer(serializers.ModelSerializer): - class Meta: - model = User - fields = ["username", "email"] - - -class VocabsBaseSerializer(serializers.HyperlinkedModelSerializer): - userAdded = serializers.HyperlinkedRelatedField( - view_name="apis:apis_api:user-detail", lookup_field="pk", read_only=True - ) - parent_class = serializers.HyperlinkedRelatedField( - view_name="apis:apis_api:vocabsbaseclass-detail", - lookup_field="pk", - read_only=True, - ) - vocab_name = serializers.HyperlinkedRelatedField( - view_name="apis:apis_api:vocabnames-detail", lookup_field="pk", read_only=True - ) - - -class CollectionTypeSerializer(VocabsBaseSerializer): - url = serializers.HyperlinkedIdentityField( - view_name="apis:apis_api:collectiontype-detail", lookup_field="pk" - ) - - class Meta: - fields = "__all__" - model = CollectionType - - -class VocabsBaseClassSerializer(VocabsBaseSerializer): - url = serializers.HyperlinkedIdentityField( - view_name="apis:apis_api:vocabsbaseclass-detail", lookup_field="pk" - ) - - class Meta: - fields = "__all__" - model = VocabsBaseClass - - -class VocabNamesSerializer(VocabsBaseSerializer): - class Meta: - fields = "__all__" - model = VocabNames - - -############################################################ -# -# Entity Types -# -########################################################### - - -# TODO RDF: Check if this should be removed or adapted -# -# class InstitutionTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:institutiontype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = InstitutionType -# -# -# class ProfessionTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:profession-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = ProfessionType -# -# -# class PlaceTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:placetype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = PlaceType -# -# -# class EventTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:eventtype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = EventType -# -# -# class WorkTypeSerializer(VocabsBaseSerializer): -# url = serializers.HyperlinkedIdentityField( -# view_name="apis:apis_api:worktype-detail", -# lookup_field="pk" -# ) -# -# class Meta: -# fields = '__all__' -# model = WorkType -# -# diff --git a/apis_core/apis_vocabularies/tests.py b/apis_core/apis_vocabularies/tests.py deleted file mode 100644 index a39b155ac..000000000 --- a/apis_core/apis_vocabularies/tests.py +++ /dev/null @@ -1 +0,0 @@ -# Create your tests here. diff --git a/apis_core/apis_vocabularies/urls.py b/apis_core/apis_vocabularies/urls.py deleted file mode 100644 index 524b89b0b..000000000 --- a/apis_core/apis_vocabularies/urls.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.urls import path, re_path - -from apis_core.apis_entities.autocomplete3 import GenericVocabulariesAutocomplete -from apis_core.apis_vocabularies import views - -app_name = "apis_vocabularies" - -urlpatterns = [ - path("download//", views.dl_vocabs_as_csv, name="dl-vocabs"), - # url(r'^autocomplete/(?P[a-zA-Z0-9-]+)/(?P[a-zA-Z0-9-]+)/$', - re_path( - r"^autocomplete/(?P[a-zA-Z0-9-]+)/$", - # url(r'^autocomplete/$', - GenericVocabulariesAutocomplete.as_view(), - name="generic_vocabularies_autocomplete", - ), -] diff --git a/apis_core/apis_vocabularies/views.py b/apis_core/apis_vocabularies/views.py deleted file mode 100644 index ad1dc9c8a..000000000 --- a/apis_core/apis_vocabularies/views.py +++ /dev/null @@ -1,25 +0,0 @@ -import pandas as pd -from django.contrib.auth.decorators import login_required -from django.contrib.contenttypes.models import ContentType -from django.http import HttpResponse - - -@login_required -def dl_vocabs_as_csv(request, model_name): - kwargs = {"app_label": "apis_vocabularies", "entity": model_name} - model = ContentType.objects.get( - app_label=kwargs.get("app_label"), model=kwargs.get("entity").lower() - ).model_class() - data = [[x.id, x.name, x.parent_class] for x in model.objects.all()] - columns = ["id", "name", "parent_class"] - df = pd.DataFrame(data, columns=columns) - filename = f"{kwargs.get('entity').lower()}.csv" - # Create the HttpResponse object with the appropriate CSV header. - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = f"attachment; filename='{filename}'" - df.to_csv(response, index=False) - - return response - - -# Create your views here. diff --git a/apis_core/testrunners.py b/apis_core/testrunners.py index 1b071c7d0..6b9bf5d5c 100644 --- a/apis_core/testrunners.py +++ b/apis_core/testrunners.py @@ -31,7 +31,6 @@ def setup_databases(self, **kwargs): for x in ct.objects.filter( app_label__in=[ "apis_metainfo", - "apis_vocabularies", "apis_entities", "apis_relations", ] @@ -40,8 +39,6 @@ def setup_databases(self, **kwargs): model__in=[ "tempentityclass", "texttype_collections", - "vocabnames", - "vocabsuri", "uricandidate", ] ) diff --git a/apis_core/urls.py b/apis_core/urls.py index 5b9a2cbf6..ce37eb223 100644 --- a/apis_core/urls.py +++ b/apis_core/urls.py @@ -13,7 +13,6 @@ # NetJsonViewSet, # PlaceGeoJsonViewSet, # ) -# from apis_core.apis_vocabularies.api_views import UserViewSet from apis_core.utils import caching from apis_core.apis_metainfo.viewsets import UriToObjectViewSet from apis_core.core.views import Dumpdata @@ -129,10 +128,6 @@ def build_apis_mock_request(method, path, view, original_request, **kwargs): "relations/", include("apis_core.apis_relations.urls", namespace="apis_relations"), ), - path( - "vocabularies/", - include("apis_core.apis_vocabularies.urls", namespace="apis_vocabularies"), - ), path( "metainfo/", include("apis_core.apis_metainfo.urls", namespace="apis_metainfo"), diff --git a/apis_core/utils/caching.py b/apis_core/utils/caching.py index 8d4a6d568..19cfc00e9 100644 --- a/apis_core/utils/caching.py +++ b/apis_core/utils/caching.py @@ -10,8 +10,6 @@ _entity_class_names = None # _reification_classes = None # _reification_class_names = None -# _vocabulary_classes = None -# _vocabulary_class_names = None _contenttype_classes = None _contenttype_class_names = None _property_autocomplete_choices = None @@ -21,7 +19,7 @@ def _init_all_ontology_classes(): """ internal function that initiales the lists containing all ontology classes (entities, - reifications, vocabularies) + reifications) :return: None """ @@ -31,7 +29,6 @@ def _init_all_ontology_classes(): from apis_core.apis_entities.models import AbstractEntity, TempEntityClass # from apis_core.apis_relations.models import AbstractReification - # from apis_core.apis_vocabularies.models import AbstractVocabulary global _ontology_classes global _ontology_class_names @@ -39,8 +36,6 @@ def _init_all_ontology_classes(): global _entity_class_names # global _reification_classes # global _reification_class_names - # global _vocabulary_classes - # global _vocabulary_class_names if ( _ontology_classes is not None or _ontology_class_names is not None @@ -48,8 +43,6 @@ def _init_all_ontology_classes(): or _entity_class_names is not None # or _reification_classes is not None # or _reification_class_names is not None - # or _vocabulary_classes is not None - # or _vocabulary_class_names is not None ): raise Exception( "initialization is called but some variables are already initialized." @@ -60,8 +53,6 @@ def _init_all_ontology_classes(): _entity_class_names = [] # _reification_classes = [] # _reification_class_names = [] - # _vocabulary_classes = [] - # _vocabulary_class_names = [] try: from apis_ontology import models as ontology_models @@ -86,18 +77,8 @@ def _init_all_ontology_classes(): # ): # _reification_classes.append(ontology_class) # _reification_class_names.append(ontology_class_name.lower()) - # elif ( - # issubclass(ontology_class, AbstractVocabulary) - # and not ontology_class._meta.abstract - # ): - # _vocabulary_classes.append(ontology_class) - # _vocabulary_class_names.append(ontology_class_name.lower()) - # _ontology_classes = _entity_classes + _reification_classes + _vocabulary_classes _ontology_classes = _entity_classes - _ontology_class_names = ( - # _entity_class_names + _reification_class_names + _vocabulary_class_names - _entity_class_names - ) + _ontology_class_names = _entity_class_names def get_all_ontology_classes(): @@ -290,7 +271,6 @@ def init_contenttype_classes(): ] apis_modules = [ "apis_core.apis_metainfo.models", - "apis_core.apis_vocabularies.models", "apis_core.apis_entities.models", "apis_core.apis_relations.models", "apis_ontology.models", diff --git a/apis_core/utils/test_caching.py b/apis_core/utils/test_caching.py index f38ee3f9c..04f782c82 100644 --- a/apis_core/utils/test_caching.py +++ b/apis_core/utils/test_caching.py @@ -8,12 +8,6 @@ "RootObject", "Uri", ], - "apis_vocabularies": [ - "CollectionType", - "VocabNames", - "VocabsBaseClass", - "VocabsUri", - ], "apis_entities": ["TempEntityClass"], "apis_relations": ["TempTriple", "Property", "Triple"], "contenttypes": ["ContentType"], diff --git a/tests/settings_ci.py b/tests/settings_ci.py index 8e9464a50..13965a9f3 100644 --- a/tests/settings_ci.py +++ b/tests/settings_ci.py @@ -14,9 +14,9 @@ "django.contrib.messages", "django.contrib.staticfiles", "apis_core.apis_metainfo", - "apis_core.apis_vocabularies", "apis_core.apis_relations", "apis_core.apis_entities", + "apis_core.apis_vocabularies", "reversion", # ui stuff "crispy_forms",