Merge pull request #832 from City-of-Helsinki/UHF-10643

UHF-10643: Add suggestions search api index.

conf/cmi/search_api.index.suggestions.yml

@@ -0,0 +1,77 @@
+uuid: 1a06b7cd-eb89-4e5d-bb5d-2bfb922b9543
+langcode: en
+status: true
+dependencies:
+  config:
+    - search_api.server.default
+  module:
+    - helfi_annif
+    - helfi_etusivu
+    - helfi_react_search
+id: suggestions
+name: Suggestions
+description: ''
+read_only: false
+field_settings:
+  keywords:
+    label: Keywords
+    datasource_id: 'entity:suggested_topics'
+    property_path: keywords_scored
+    type: scored_item
+  parent_bundle:
+    label: 'Parent bundle'
+    datasource_id: 'entity:suggested_topics'
+    property_path: parent_bundle
+    type: string
+    dependencies:
+      module:
+        - helfi_annif
+  parent_id:
+    label: 'Parent ID'
+    datasource_id: 'entity:suggested_topics'
+    property_path: parent_id
+    type: string
+    dependencies:
+      module:
+        - helfi_annif
+  parent_instance:
+    label: 'Parent instance'
+    datasource_id: 'entity:suggested_topics'
+    property_path: parent_instance
+    type: string
+    dependencies:
+      module:
+        - helfi_annif
+  parent_type:
+    label: 'Parent type'
+    datasource_id: 'entity:suggested_topics'
+    property_path: parent_type
+    type: string
+    dependencies:
+      module:
+        - helfi_annif
+datasource_settings:
+  'entity:suggested_topics': {  }
+processor_settings:
+  add_url: {  }
+  aggregated_field: {  }
+  custom_value: {  }
+  district_image_absolute_url: {  }
+  entity_type: {  }
+  language_with_fallback: {  }
+  main_image_url: {  }
+  project_execution_schedule: {  }
+  project_image_absolute_url: {  }
+  project_plan_schedule: {  }
+  rendered_item: {  }
+  scored_reference: {  }
+  uuid_langcode: {  }
+tracker_settings:
+  default:
+    indexing_order: fifo
+options:
+  cron_limit: 50
+  delete_on_fail: true
+  index_directly: true
+  track_changes_in_references: true
+server: default

public/modules/custom/helfi_annif/README.md

@@ -6,6 +6,12 @@ Keywords are generated for entities that have `field_annif_keywords` with `hook_
 
 See the API documentation at: [https://ai.finto.fi/v1/ui/](https://ai.finto.fi/v1/ui/).
 
+## Suggestions across hel.fi instances
+
+**This feature is work in progress.**
+
+Suggested topics entities are synced to suggestions search api index. In the future, other instances communicate with etusivu so that suggested topics entity is created for their content, and they can search suggestions for their content from the shared search index.
+
 ## Text converter
 
 The AI API accepts raw text only. Drupal content must be converted to UTF-8 encoded raw text. This is achieved with `TextConverterManager`.

public/modules/custom/helfi_annif/config/schema/helfi_annif.schema.yml

@@ -18,3 +18,19 @@ field.value.suggested_topics_reference:
     value:
       type: label
       label: Value
+
+field.storage_settings.scored_entity_reference:
+  type: field.storage_settings.entity_reference
+  label: 'Scored reference field storage settings'
+
+field.field_settings.scored_entity_reference:
+  type: field.field_settings.entity_reference
+  label: 'Scored reference settings'
+
+field.storage_settings.suggested_topics_reference:
+  type: field.storage_settings.entity_reference
+  label: 'Suggested topics field storage settings'
+
+field.field_settings.suggested_topics_reference:
+  type: field.field_settings.entity_reference
+  label: 'Suggested topics reference settings'

public/modules/custom/helfi_annif/helfi_annif.install

@@ -7,6 +7,9 @@
 
 declare(strict_types=1);
 
+use Drupal\Core\Entity\EntityTypeInterface;
+use Drupal\Core\Field\BaseFieldDefinition;
+
 /**
  * Create entity definitions for annif-fields.
  */
@@ -96,3 +99,29 @@ function helfi_annif_update_9003(): void {
 
   // @todo remove obsolete field annif_keywords in a future update hook.
 }
+
+/**
+ * Updates entity fields for suggested_topics.
+ */
+function helfi_annif_update_9004(): void {
+  $fields['parent_id'] = BaseFieldDefinition::create('string')
+    ->setLabel(t('Parent ID'))
+    ->setDescription(t('The ID of the parent entity of which this entity is referenced.'))
+    ->setSetting('is_ascii', TRUE);
+
+  $fields['parent_type'] = BaseFieldDefinition::create('string')
+    ->setLabel(t('Parent type'))
+    ->setDescription(t('The entity parent type to which this entity is referenced.'))
+    ->setSetting('is_ascii', TRUE)
+    ->setSetting('max_length', EntityTypeInterface::ID_MAX_LENGTH);
+
+  $fields['parent_instance'] = BaseFieldDefinition::create('string')
+    ->setLabel(t('Parent instance'))
+    ->setDescription(t('The name of the instance where this entity is located at.'))
+    ->setSetting('is_ascii', TRUE);
+
+  foreach ($fields as $name => $field) {
+    \Drupal::entityDefinitionUpdateManager()
+      ->installFieldStorageDefinition($name, 'suggested_topics', 'helfi_annif', $field);
+  }
+}

public/modules/custom/helfi_annif/helfi_annif.services.yml

@@ -22,3 +22,5 @@ services:
   Drupal\helfi_annif\TextConverter\RenderTextConverter:
     tags:
       - { name: helfi_annif.text_converter, priority: -1 }
+
+  Drupal\helfi_annif\EventSubscriber\SearchApiSubscriber: ~

public/modules/custom/helfi_annif/src/Entity/SuggestedTopics.php

@@ -67,6 +67,28 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type): a
       ])
       ->setDisplayConfigurable('view', TRUE);
 
+    $fields['parent_id'] = BaseFieldDefinition::create('string')
+      ->setLabel(t('Parent ID'))
+      ->setDescription(t('The ID of the parent entity of which this entity is referenced.'))
+      ->setSetting('is_ascii', TRUE);
+
+    $fields['parent_type'] = BaseFieldDefinition::create('string')
+      ->setLabel(t('Parent type'))
+      ->setDescription(t('The entity parent type to which this entity is referenced.'))
+      ->setSetting('is_ascii', TRUE)
+      ->setSetting('max_length', EntityTypeInterface::ID_MAX_LENGTH);
+
+    $fields['parent_bundle'] = BaseFieldDefinition::create('string')
+      ->setLabel(t('Parent bundle'))
+      ->setDescription(t('The entity parent bundle to which this entity is referenced.'))
+      ->setSetting('is_ascii', TRUE)
+      ->setSetting('max_length', EntityTypeInterface::ID_MAX_LENGTH);
+
+    $fields['parent_instance'] = BaseFieldDefinition::create('string')
+      ->setLabel(t('Parent instance'))
+      ->setDescription(t('The name of the instance where this entity is located at.'))
+      ->setSetting('is_ascii', TRUE);
+
     return $fields;
   }
 

public/modules/custom/helfi_annif/src/EventSubscriber/SearchApiSubscriber.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\helfi_annif\EventSubscriber;
+
+use Drupal\search_api\Event\MappingFieldTypesEvent;
+use Drupal\search_api\Event\SearchApiEvents;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+/**
+ * Search api event subscriber.
+ */
+final class SearchApiSubscriber implements EventSubscriberInterface {
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function getSubscribedEvents(): array {
+    return [
+      SearchApiEvents::MAPPING_FIELD_TYPES => 'mapFieldTypes',
+    ];
+  }
+
+  /**
+   * Map custom field types.
+   */
+  public function mapFieldTypes(MappingFieldTypesEvent $event): void {
+    $mapping = &$event->getFieldTypeMapping();
+    $mapping['scored_item'] = 'scored_item';
+  }
+
+}

public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/ScoredEntityReferenceItem.php

@@ -15,6 +15,8 @@
 /**
  * Defines the 'scored_entity_reference' field type.
  *
+ * @property mixed $score Item score.
+ *
  * @see \Drupal\Core\Field\Plugin\Field\FieldType\EntityReferenceItem
  * @see \Drupal\Core\Field\Plugin\Field\FieldType\FloatItem
  */

public/modules/custom/helfi_annif/src/Plugin/Field/FieldType/SuggestedTopicsReferenceItem.php

@@ -9,6 +9,7 @@
 use Drupal\Core\Field\Plugin\Field\FieldType\EntityReferenceItem;
 use Drupal\Core\Form\FormStateInterface;
 use Drupal\Core\StringTranslation\TranslatableMarkup;
+use Drupal\helfi_annif\Entity\SuggestedTopics;
 
 /**
  * Defines the 'suggested_topics_reference' field type.
@@ -58,12 +59,16 @@ public function fieldSettingsForm(array $form, FormStateInterface $form_state):
   /**
    * {@inheritdoc}
    */
-  public function preSave() {
-    if ($this->hasNewEntity()) {
-      $this->entity->save();
-    }
+  public function postSave($update): bool {
+    $parent = $this->getEntity();
+    $entity = $this->entity;
+    assert($entity instanceof SuggestedTopics);
+
+    $entity
+      ->set('parent_id', $parent->id())
+      ->save();
 
-    parent::preSave();
+    return FALSE;
   }
 
   /**

public/modules/custom/helfi_annif/src/Plugin/Field/FieldWidget/SuggestedTopicsReferenceWidget.php

@@ -11,6 +11,7 @@
 use Drupal\Core\Form\FormStateInterface;
 use Drupal\Core\StringTranslation\TranslatableMarkup;
 use Drupal\helfi_annif\Entity\SuggestedTopics;
+use Drupal\helfi_api_base\Environment\Project;
 
 /**
  * Defines the 'suggested_topics_reference' field widget.
@@ -37,7 +38,11 @@ public function formElement(FieldItemListInterface $items, $delta, array $elemen
     $hasTargetEntity = !empty($items[$delta]->target_id) && $items[$delta]->entity;
 
     /** @var \Drupal\helfi_annif\Entity\SuggestedTopics $entity */
-    $entity = $hasTargetEntity ? $items[$delta]->entity : SuggestedTopics::create([]);
+    $entity = $hasTargetEntity ? $items[$delta]->entity : SuggestedTopics::create([
+      'parent_type' => $items->getEntity()->getEntityTypeId(),
+      'parent_bundle' => $items->getEntity()->bundle(),
+      'parent_instance' => Project::ETUSIVU,
+    ]);
 
     $element['entity'] = [
       '#type' => 'value',

public/modules/custom/helfi_annif/src/Plugin/search_api/data_type/ScoredItemDataType.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\helfi_annif\Plugin\search_api\data_type;
+
+use Drupal\search_api\DataType\DataTypePluginBase;
+
+/**
+ * Provides a string data type.
+ *
+ * @SearchApiDataType(
+ *   id = "scored_item",
+ *   label = @Translation("Scored item"),
+ *   description = @Translation("Item with score."),
+ *   fallback_type = "string"
+ * )
+ */
+class ScoredItemDataType extends DataTypePluginBase {
+}

public/modules/custom/helfi_annif/src/Plugin/search_api/processor/ScoredReferenceProcessor.php

@@ -0,0 +1,90 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\helfi_annif\Plugin\search_api\processor;
+
+use Drupal\Core\Entity\ContentEntityInterface;
+use Drupal\Core\Field\FieldDefinitionInterface;
+use Drupal\helfi_annif\Plugin\Field\FieldType\ScoredEntityReferenceItem;
+use Drupal\search_api\Datasource\DatasourceInterface;
+use Drupal\search_api\Item\ItemInterface;
+use Drupal\search_api\Processor\ProcessorPluginBase;
+use Drupal\search_api\Processor\ProcessorProperty;
+
+/**
+ * Indexes uuid with langcode.
+ *
+ * @SearchApiProcessor(
+ *   id = "scored_reference",
+ *   label = @Translation("Scored references"),
+ *   description = @Translation("Indexes scored references"),
+ *   stages = {
+ *     "add_properties" = 0
+ *   },
+ *   locked = true,
+ *   hidden = true,
+ * )
+ */
+final class ScoredReferenceProcessor extends ProcessorPluginBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getPropertyDefinitions(?DataSourceInterface $datasource = NULL) : array {
+    $properties = [];
+
+    if ($datasource) {
+      $propertyDefinitions = $datasource->getPropertyDefinitions();
+      foreach ($propertyDefinitions as $id => $definition) {
+        if (
+          $definition instanceof FieldDefinitionInterface &&
+          $definition->getType() === 'scored_entity_reference'
+        ) {
+          $properties[$id . '_scored'] = new ProcessorProperty([
+            'label' => $this->t('Scored reference'),
+            'description' => $this->t('Indexes referenced item labels with a score'),
+            'type' => 'scored_item',
+            'processor_id' => $this->getPluginId(),
+          ]);
+        }
+      }
+    }
+
+    return $properties;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function addFieldValues(Iteminterface $item) : void {
+    $entity = $item->getOriginalObject()?->getValue();
+
+    if (!$entity instanceof ContentEntityInterface) {
+      return;
+    }
+
+    $objectSupport = $this->index->getServerInstance()->getBackendId() === 'elasticsearch';
+
+    foreach ($item->getFields() as $field) {
+      if ($field->getOriginalType() !== 'scored_item') {
+        continue;
+      }
+
+      $property = substr($field->getPropertyPath(), 0, -strlen("_scored"));
+
+      $scoredReferenceField = $entity->get($property);
+      foreach ($scoredReferenceField as $scoredReference) {
+        assert($scoredReference instanceof ScoredEntityReferenceItem);
+
+        $value = [
+          'score' => (float) $scoredReference->score,
+          'label' => $scoredReference->entity->id(),
+        ];
+
+        $field->addValue($objectSupport ? $value : json_encode($value));
+      }
+    }
+  }
+
+}