diff --git a/core/app/datasource/src/main/java/io/openk9/datasource/listener/K9EntityListener.java b/core/app/datasource/src/main/java/io/openk9/datasource/listener/K9EntityListener.java index c1c09cac2..b6e14f745 100644 --- a/core/app/datasource/src/main/java/io/openk9/datasource/listener/K9EntityListener.java +++ b/core/app/datasource/src/main/java/io/openk9/datasource/listener/K9EntityListener.java @@ -42,7 +42,6 @@ public void beforeUpdate(K9Entity k9Entity) { _handle(k9Entity, EventType.UPDATE); } - @PostRemove public void postRemove(K9Entity k9Entity) { _handle(k9Entity, EventType.DELETE); diff --git a/core/app/datasource/src/main/java/io/openk9/datasource/model/DataIndex.java b/core/app/datasource/src/main/java/io/openk9/datasource/model/DataIndex.java index ecb61776d..fd37a57ac 100644 --- a/core/app/datasource/src/main/java/io/openk9/datasource/model/DataIndex.java +++ b/core/app/datasource/src/main/java/io/openk9/datasource/model/DataIndex.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import io.openk9.datasource.model.util.K9Entity; +import io.openk9.datasource.util.OpenSearchUtils; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -35,7 +37,10 @@ import javax.persistence.ManyToMany; import javax.persistence.ManyToOne; import javax.persistence.OneToOne; +import javax.persistence.PostLoad; +import javax.persistence.PostUpdate; import javax.persistence.Table; +import javax.persistence.Transient; @Entity @Table(name = "data_index") @@ -75,6 +80,11 @@ public class DataIndex extends K9Entity { @JoinColumn(name = "vector_index_id", referencedColumnName = "id") private VectorIndex vectorIndex; + @Transient + @Setter(AccessLevel.NONE) + @Getter(AccessLevel.NONE) + private String indexName; + public void addDocType(DocType docType) { docTypes.add(docType); } @@ -83,8 +93,45 @@ public void removeDocType(DocType docType) { docTypes.remove(docType); } - public String getIndexName() { - return getTenant() + "-" + name; + public String getIndexName() throws UnknownTenantException { + if (indexName == null) { + setupIndexName(); + } + + return indexName; } + @PostLoad + @PostUpdate + protected void setupIndexName() throws UnknownTenantException { + String tenantId = getTenant(); + + // This is a workaround needed when a new DataIndex is being created. + // The tenant is not identified, likely because the entity has not + // been persisted yet. Therefore, it is obtained from an entity that + // is already in the persistence context, typically, the first + // DocType associated with the new DataIndex, or alternatively, + // the Datasource associated with it. + if (tenantId == null) { + var iterator = docTypes.iterator(); + if (iterator.hasNext()) { + var docType = iterator.next(); + tenantId = docType.getTenant(); + } + else { + var ds = getDatasource(); + if (ds != null) { + tenantId = ds.getTenant(); + } + } + if (tenantId == null) { + throw new UnknownTenantException( + String.format("Cannot identify the tenant for DataIndex: %s", getName())); + } + } + + this.indexName = OpenSearchUtils.indexNameSanitizer( + String.format("%s-%s", tenantId, getName()) + ); + } } \ No newline at end of file diff --git a/core/app/datasource/src/main/java/io/openk9/datasource/model/UnknownTenantException.java b/core/app/datasource/src/main/java/io/openk9/datasource/model/UnknownTenantException.java new file mode 100644 index 000000000..55366fe1e --- /dev/null +++ b/core/app/datasource/src/main/java/io/openk9/datasource/model/UnknownTenantException.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2020-present SMC Treviso s.r.l. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.openk9.datasource.model; + +import io.openk9.datasource.service.exception.K9Error; + +public class UnknownTenantException extends K9Error { + + public UnknownTenantException(String message) { + super(message); + } + + public UnknownTenantException(String message, Throwable cause) { + super(message, cause); + } + + public UnknownTenantException(Throwable cause) { + super(cause); + } + +} diff --git a/core/app/datasource/src/main/java/io/openk9/datasource/service/DataIndexService.java b/core/app/datasource/src/main/java/io/openk9/datasource/service/DataIndexService.java index cb18015e2..030624b86 100644 --- a/core/app/datasource/src/main/java/io/openk9/datasource/service/DataIndexService.java +++ b/core/app/datasource/src/main/java/io/openk9/datasource/service/DataIndexService.java @@ -20,12 +20,15 @@ import io.openk9.common.graphql.util.relay.Connection; import io.openk9.common.util.SortBy; import io.openk9.datasource.index.IndexService; +import io.openk9.datasource.index.mappings.MappingsKey; +import io.openk9.datasource.index.mappings.MappingsUtil; import io.openk9.datasource.mapper.DataIndexMapper; import io.openk9.datasource.mapper.IngestionPayloadMapper; import io.openk9.datasource.model.DataIndex; import io.openk9.datasource.model.DataIndex_; import io.openk9.datasource.model.Datasource; import io.openk9.datasource.model.DocType; +import io.openk9.datasource.model.UnknownTenantException; import io.openk9.datasource.model.VectorIndex; import io.openk9.datasource.model.dto.DataIndexDTO; import io.openk9.datasource.plugindriver.HttpPluginDriverClient; @@ -39,19 +42,32 @@ import io.openk9.datasource.util.OpenSearchUtils; import io.smallrye.mutiny.Uni; import io.vertx.core.json.Json; +import io.vertx.core.json.JsonObject; import org.hibernate.reactive.mutiny.Mutiny; +import org.opensearch.OpenSearchStatusException; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.IndicesClient; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.PutComposableIndexTemplateRequest; +import org.opensearch.cluster.metadata.ComposableIndexTemplate; +import org.opensearch.cluster.metadata.Template; +import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.settings.Settings; import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; @ApplicationScoped public class DataIndexService @@ -70,6 +86,8 @@ public class DataIndexService @Inject IngestionPayloadMapper ingestionPayloadMapper; + private static final String DETAILS_FIELD = "details"; + DataIndexService(DataIndexMapper mapper) { this.mapper = mapper; } @@ -212,18 +230,18 @@ public Uni deleteById(long entityId) { .transformToUni(dataIndex -> Uni.createFrom() .emitter(emitter -> { - DeleteIndexRequest deleteIndexRequest = - new DeleteIndexRequest(dataIndex.getIndexName()); + try { + DeleteIndexRequest deleteIndexRequest = + new DeleteIndexRequest(dataIndex.getIndexName()); - deleteIndexRequest - .indicesOptions( - IndicesOptions.fromMap( - Map.of("ignore_unavailable", true), - deleteIndexRequest.indicesOptions() - ) - ); + deleteIndexRequest + .indicesOptions( + IndicesOptions.fromMap( + Map.of("ignore_unavailable", true), + deleteIndexRequest.indicesOptions() + ) + ); - try { AcknowledgedResponse delete = restHighLevelClient.indices().delete( deleteIndexRequest, RequestOptions.DEFAULT @@ -231,7 +249,7 @@ public Uni deleteById(long entityId) { emitter.complete(delete); } - catch (IOException e) { + catch (UnknownTenantException | IOException e) { emitter.fail(e); } }) @@ -279,4 +297,122 @@ public Uni createByDatasource(Mutiny.Session session, Datasource data }); } + public Uni createDataIndexFromDocTypes( + long datasourceId, List docTypeIds, String name, + Map indexSettings) { + + String dataIndexName = name == null ? "data-" + OffsetDateTime.now() : name; + + return sessionFactory.withTransaction((s, t) -> docTypeService + .findDocTypes(docTypeIds, s) + .flatMap(docTypeList -> { + + if (docTypeList.size() != docTypeIds.size()) { + throw new RuntimeException( + "docTypeIds found: " + docTypeList.size() + + " docTypeIds requested: " + docTypeIds.size()); + } + + DataIndex dataIndex = new DataIndex(); + + dataIndex.setDescription("auto-generated"); + + dataIndex.setName(dataIndexName); + + dataIndex.setDocTypes(new LinkedHashSet<>(docTypeList)); + + dataIndex.setDatasource(s.getReference(Datasource.class, datasourceId)); + + return persist(s, dataIndex) + .map(__ -> { + Map mappings = + MappingsUtil.docTypesToMappings(dataIndex.getDocTypes()); + + Settings settings; + + Map settingsMap = + indexSettings != null && !indexSettings.isEmpty() ? + indexSettings : + MappingsUtil.docTypesToSettings(dataIndex.getDocTypes()); + + if (settingsMap.isEmpty()) { + settings = Settings.EMPTY; + } + else { + settings = Settings.builder() + .loadFromMap(settingsMap) + .build(); + } + + PutComposableIndexTemplateRequest + putComposableIndexTemplateRequest = + new PutComposableIndexTemplateRequest(); + + ComposableIndexTemplate composableIndexTemplate = null; + + try { + var indexName = dataIndex.getIndexName(); + + composableIndexTemplate = new ComposableIndexTemplate( + List.of(indexName), + new Template(settings, new CompressedXContent( + Json.encode(mappings)), null), + null, null, null, null + ); + + putComposableIndexTemplateRequest + .name(indexName + "-template") + .indexTemplate(composableIndexTemplate); + + return putComposableIndexTemplateRequest; + } + catch (UnknownTenantException e) { + throw new WebApplicationException(Response + .status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(JsonObject.of( + DETAILS_FIELD, "cannot obtain a proper index name" + )) + .build()); + } + catch (IOException e) { + throw new WebApplicationException(Response + .status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(JsonObject.of( + DETAILS_FIELD, "failed creating IndexTemplate" + )) + .build()); + } + + }) + .call((req) -> Uni.createFrom().emitter((sink) -> { + + try { + IndicesClient indices = restHighLevelClient.indices(); + + indices.putIndexTemplate(req, RequestOptions.DEFAULT); + + sink.complete(null); + } + catch (OpenSearchStatusException e) { + sink.fail(new WebApplicationException(javax.ws.rs.core.Response + .status(e.status().getStatus()) + .entity(JsonObject.of( + DETAILS_FIELD, e.getMessage())) + .build())); + } + catch (Exception e) { + sink.fail(new WebApplicationException(javax.ws.rs.core.Response + .status(javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR) + .entity(JsonObject.of( + DETAILS_FIELD, e.getMessage())) + .build())); + } + + })) + .map(__ -> dataIndex); + + }) + ); + } + } diff --git a/core/app/datasource/src/main/java/io/openk9/datasource/service/DocTypeService.java b/core/app/datasource/src/main/java/io/openk9/datasource/service/DocTypeService.java index fd6f778d0..e80aa03cb 100644 --- a/core/app/datasource/src/main/java/io/openk9/datasource/service/DocTypeService.java +++ b/core/app/datasource/src/main/java/io/openk9/datasource/service/DocTypeService.java @@ -19,6 +19,8 @@ import io.openk9.common.graphql.util.relay.Connection; import io.openk9.common.util.SortBy; +import io.openk9.datasource.index.mappings.MappingsKey; +import io.openk9.datasource.index.mappings.MappingsUtil; import io.openk9.datasource.mapper.DocTypeFieldMapper; import io.openk9.datasource.mapper.DocTypeMapper; import io.openk9.datasource.model.AclMapping; @@ -42,6 +44,11 @@ import org.hibernate.FlushMode; import org.hibernate.reactive.mutiny.Mutiny; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; import javax.persistence.criteria.CriteriaBuilder; @@ -53,9 +60,6 @@ import javax.persistence.criteria.Root; import javax.persistence.criteria.SetJoin; import javax.persistence.criteria.Subquery; -import java.util.Collection; -import java.util.List; -import java.util.Set; @ApplicationScoped public class DocTypeService extends BaseK9EntityService { @@ -332,6 +336,27 @@ public Uni deleteById(long entityId) { ); } + public Uni> findDocTypes(List docTypeIds, Mutiny.Session session) { + + return getDocTypesAndDocTypeFields(session, docTypeIds).map(LinkedList::new); + } + + + public Uni> findDocTypes(List docTypeIds) { + + return sessionFactory.withSession(s -> findDocTypes(docTypeIds, s)); + } + + public Uni> getMappingsFromDocTypes(List docTypeIds) { + + return findDocTypes(docTypeIds).map(MappingsUtil::docTypesToMappings); + } + + public Uni> getSettingsFromDocTypes(List docTypeIds) { + + return findDocTypes(docTypeIds).map(MappingsUtil::docTypesToSettings); + } + @Inject DocTypeFieldService docTypeFieldService; diff --git a/core/app/datasource/src/main/java/io/openk9/datasource/util/OpenSearchUtils.java b/core/app/datasource/src/main/java/io/openk9/datasource/util/OpenSearchUtils.java index 3181e3ff5..2889f50df 100644 --- a/core/app/datasource/src/main/java/io/openk9/datasource/util/OpenSearchUtils.java +++ b/core/app/datasource/src/main/java/io/openk9/datasource/util/OpenSearchUtils.java @@ -48,6 +48,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; public class OpenSearchUtils { @@ -107,6 +109,9 @@ public static JsonObject getDynamicMapping(byte[] payload) { return mappings; } + private static final Set FORBIDDEN_CHARACTERS = Set.of( + ':', '#', '\\', '/', '*', '?', '"', '<', '>', '|', ' ', ','); + public static WrapperQueryBuilder toWrapperQueryBuilder(Query query) { try (var os = new ByteArrayOutputStream()) { @@ -193,4 +198,39 @@ private static IndexSettings getIndexSettings() { return indexSettings; } + /** + * Take a candidate indexName and removes illegal first characters + * then replace all forbidden characters (include uppercase). + * + * @param name the candidate name + * @return the sanitized name + */ + public static String indexNameSanitizer(String name) { + Objects.requireNonNull(name); + + var n = name.length(); + + if (n == 0) { + throw new IllegalArgumentException("name is empty"); + } + + StringBuilder builder = new StringBuilder(); + + int i = 0; + while (name.charAt(i) == '_' || name.charAt(i) == '-') { + i++; + } + + for (; i < n; i++) { + if (FORBIDDEN_CHARACTERS.contains(name.charAt(i))) { + builder.append("_"); + } + else { + builder.append(name.charAt(i)); + } + } + + return builder.toString().toLowerCase(); + } + } diff --git a/core/app/datasource/src/main/java/io/openk9/datasource/web/DataIndexResource.java b/core/app/datasource/src/main/java/io/openk9/datasource/web/DataIndexResource.java index 077d81896..b8a6a8264 100644 --- a/core/app/datasource/src/main/java/io/openk9/datasource/web/DataIndexResource.java +++ b/core/app/datasource/src/main/java/io/openk9/datasource/web/DataIndexResource.java @@ -18,35 +18,20 @@ package io.openk9.datasource.web; import io.openk9.datasource.index.mappings.MappingsKey; -import io.openk9.datasource.index.mappings.MappingsUtil; import io.openk9.datasource.model.DataIndex; import io.openk9.datasource.model.Datasource; import io.openk9.datasource.model.Datasource_; -import io.openk9.datasource.model.DocType; import io.openk9.datasource.processor.indexwriter.IndexerEvents; +import io.openk9.datasource.service.DataIndexService; import io.openk9.datasource.service.DocTypeService; import io.smallrye.mutiny.Uni; -import io.smallrye.mutiny.unchecked.Unchecked; -import io.vertx.core.json.Json; -import io.vertx.core.json.JsonObject; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.eclipse.microprofile.faulttolerance.CircuitBreaker; import org.hibernate.reactive.mutiny.Mutiny; -import org.opensearch.OpenSearchStatusException; -import org.opensearch.client.IndicesClient; -import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; -import org.opensearch.client.indices.PutComposableIndexTemplateRequest; -import org.opensearch.cluster.metadata.ComposableIndexTemplate; -import org.opensearch.cluster.metadata.Template; -import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.settings.Settings; -import java.time.OffsetDateTime; -import java.util.LinkedHashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import javax.annotation.security.RolesAllowed; @@ -57,14 +42,49 @@ import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; @CircuitBreaker @Path("/v1/data-index") @RolesAllowed("k9-admin") public class DataIndexResource { + @Inject + RestHighLevelClient restHighLevelClient; + @Inject + Mutiny.SessionFactory sessionFactory; + @Inject + IndexerEvents indexerEvents; + @Inject + DocTypeService docTypeService; + @Inject + DataIndexService dataIndexService; + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class AutoGenerateDocTypesRequest { + private long datasourceId; + + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class GetMappingsOrSettingsFromDocTypesRequest { + private List docTypeIds; + + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class CreateDataIndexFromDocTypesRequest { + private List docTypeIds; + private String indexName; + private Map settings; + + } + @Path("/auto-generate-doc-types") @POST public Uni autoGenerateDocTypes( @@ -88,8 +108,7 @@ public Uni autoGenerateDocTypes( .onItem() .transformToUni( dataIndex -> indexerEvents.generateDocTypeFields( - session, - dataIndex)); + session, dataIndex)); }); @@ -100,8 +119,7 @@ public Uni autoGenerateDocTypes( public Uni> getMappings( GetMappingsOrSettingsFromDocTypesRequest request) { - return getMappingsFromDocTypes(request.getDocTypeIds()); - + return docTypeService.getMappingsFromDocTypes(request.getDocTypeIds()); } @Path("/get-settings-from-doc-types") @@ -109,59 +127,7 @@ public Uni> getMappings( public Uni> getSettings( GetMappingsOrSettingsFromDocTypesRequest request) { - return getSettingsFromDocTypes(request.getDocTypeIds()); - - } - - @Inject - RestHighLevelClient restHighLevelClient; - - private Uni> getMappingsFromDocTypes( - List docTypeIds) { - - return sessionFactory.withTransaction(session -> { - - Uni> docTypeListUni = - _findDocTypes(docTypeIds, session, false); - - return docTypeListUni.map(MappingsUtil::docTypesToMappings); - - }); - } - - private Uni> getSettingsFromDocTypes( - List docTypeIds) { - - return sessionFactory.withTransaction(session -> { - - Uni> docTypeListUni = - _findDocTypes(docTypeIds, session, true); - - return docTypeListUni.map(MappingsUtil::docTypesToSettings); - - }); - } - - private Uni> _findDocTypes( - List docTypeIds, Mutiny.Session session, boolean includeAnalyzerSubtypes) { - - return docTypeService - .getDocTypesAndDocTypeFields(session, docTypeIds) - .map(LinkedList::new); - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class AutoGenerateDocTypesRequest { - private long datasourceId; - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class GetMappingsOrSettingsFromDocTypesRequest { - private List docTypeIds; + return docTypeService.getSettingsFromDocTypes(request.getDocTypeIds()); } @Path("/create-data-index-from-doc-types/{datasourceId}") @@ -170,133 +136,13 @@ public Uni createDataIndexFromDocTypes( @PathParam("datasourceId") long datasourceId, CreateDataIndexFromDocTypesRequest request) { - String indexName; - - if (request.getIndexName() == null) { - indexName = "data-" + OffsetDateTime.now(); - } - else { - indexName = request.getIndexName(); - } - - return sessionFactory.withTransaction(s -> { - - List docTypeIds = request.getDocTypeIds(); - - Uni> docTypeListUni = - _findDocTypes(docTypeIds, s, true); - - return docTypeListUni - .onItem() - .transformToUni(Unchecked.function(docTypeList -> { - - if (docTypeList.size() != docTypeIds.size()) { - throw new RuntimeException( - "docTypeIds found: " + docTypeList.size() + - " docTypeIds requested: " + docTypeIds.size()); - } - - DataIndex dataIndex = new DataIndex(); - - dataIndex.setDescription("auto-generated"); - - dataIndex.setName(indexName); - - dataIndex.setDocTypes(new LinkedHashSet<>(docTypeList)); - - dataIndex.setDatasource(s.getReference(Datasource.class, datasourceId)); - - return s.persist(dataIndex) - .map(__ -> dataIndex) - .call(s::flush) - .call((di) -> Uni.createFrom().emitter((sink) -> { - - try { - IndicesClient indices = restHighLevelClient.indices(); - - Map mappings = - MappingsUtil.docTypesToMappings(di.getDocTypes()); - - Settings settings; - - Map settingsMap = null; + return dataIndexService.createDataIndexFromDocTypes( + datasourceId, + request.getDocTypeIds(), + request.getIndexName(), + request.getSettings() + ); - Map requestSettings = request.getSettings(); - - if (requestSettings != null && !requestSettings.isEmpty()) { - settingsMap = requestSettings; - } - else { - settingsMap = MappingsUtil.docTypesToSettings(di.getDocTypes()); - } - - if (settingsMap.isEmpty()) { - settings = Settings.EMPTY; - } - else { - settings = Settings.builder() - .loadFromMap(settingsMap) - .build(); - } - - PutComposableIndexTemplateRequest - putComposableIndexTemplateRequest = - new PutComposableIndexTemplateRequest(); - - ComposableIndexTemplate composableIndexTemplate = - new ComposableIndexTemplate( - List.of(indexName), - new Template(settings, new CompressedXContent( - Json.encode(mappings)), null), - null, null, null, null); - - putComposableIndexTemplateRequest - .name(indexName + "-template") - .indexTemplate(composableIndexTemplate); - - indices.putIndexTemplate(putComposableIndexTemplateRequest, RequestOptions.DEFAULT); - - sink.complete(null); - } - catch (OpenSearchStatusException e) { - sink.fail(new WebApplicationException(Response - .status(e.status().getStatus()) - .entity(JsonObject.of( - "details", e.getMessage())) - .build())); - } - catch (Exception e) { - sink.fail(new WebApplicationException(Response - .status(Response.Status.INTERNAL_SERVER_ERROR) - .entity(JsonObject.of( - "details", e.getMessage())) - .build())); - } - - })); - - })); - - }); - - } - - @Inject - Mutiny.SessionFactory sessionFactory; - - @Inject - IndexerEvents indexerEvents; - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class CreateDataIndexFromDocTypesRequest { - private List docTypeIds; - private String indexName; - private Map settings; } - @Inject - DocTypeService docTypeService; - } diff --git a/core/app/datasource/src/test/java/io/openk9/datasource/util/OpenSearchUtilsTest.java b/core/app/datasource/src/test/java/io/openk9/datasource/util/OpenSearchUtilsTest.java index 8d6c86d0c..f75e5cf80 100644 --- a/core/app/datasource/src/test/java/io/openk9/datasource/util/OpenSearchUtilsTest.java +++ b/core/app/datasource/src/test/java/io/openk9/datasource/util/OpenSearchUtilsTest.java @@ -23,10 +23,14 @@ import io.vertx.core.json.Json; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mapstruct.factory.Mappers; import java.io.IOException; import java.io.InputStream; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -38,6 +42,21 @@ class OpenSearchUtilsTest { private static final IngestionPayloadMapper mapper = Mappers.getMapper(IngestionPayloadMapper.class); + @ParameterizedTest + @MethodSource("indexNames") + void should_sanitize_indexName(String actual, String expected) { + assertEquals(expected, OpenSearchUtils.indexNameSanitizer(actual)); + } + + private static Stream indexNames() { + return Stream.of( + Arguments.of("---aaa", "aaa"), + Arguments.of("---AAA-aaa", "aaa-aaa"), + Arguments.of("___Aaa", "aaa"), + Arguments.of("---A**AA", "a__aa") + ); + } + @Test void should_create_dynamic_mapping_for_json() throws IOException { try (InputStream in = TestUtils.getResourceAsStream("plugindriver/sample.json")) { diff --git a/core/app/searcher/src/main/java/io/openk9/searcher/resource/SearchResource.java b/core/app/searcher/src/main/java/io/openk9/searcher/resource/SearchResource.java index 8fba01f74..944632d5a 100644 --- a/core/app/searcher/src/main/java/io/openk9/searcher/resource/SearchResource.java +++ b/core/app/searcher/src/main/java/io/openk9/searcher/resource/SearchResource.java @@ -19,6 +19,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.ProtocolStringList; +import io.openk9.searcher.client.dto.ParserSearchToken; import io.openk9.searcher.client.dto.SearchRequest; import io.openk9.searcher.client.mapper.SearcherMapper; import io.openk9.searcher.grpc.QueryAnalysisRequest; @@ -88,6 +89,31 @@ @RequestScoped public class SearchResource { + private static final Logger logger = Logger.getLogger(SearchResource.class); + private static final Object namedXContentRegistryKey = new Object(); + private static final Pattern i18nHighlithKeyPattern = Pattern.compile( + "\\.i18n\\..{5,}$|\\.base$"); + private final Map namedXContentRegistryMap = + Collections.synchronizedMap(new IdentityHashMap<>()); + @GrpcClient("searcher") + Searcher searcherClient; + @Inject + @Claim(standard = Claims.raw_token) + String rawToken; + @Inject + SearcherMapper searcherMapper; + @Inject + InternalSearcherMapper internalSearcherMapper; + @Inject + RestHighLevelClient restHighLevelClient; + @Context + HttpServerRequest request; + @Context + HttpHeaders + headers; + @ConfigProperty(name = "openk9.searcher.supported.headers.name", defaultValue = "OPENK9_ACL") + List supportedHeadersName; + @POST @Path("/search-query") @Produces(MediaType.APPLICATION_JSON) @@ -109,9 +135,6 @@ public Uni searchQuery(SearchRequest searchRequest) { } - @Inject - RestHighLevelClient restHighLevelClient; - @POST @Path("/suggestions") @Produces(MediaType.APPLICATION_JSON) @@ -126,6 +149,75 @@ public Uni suggestions(SearchRequest searchRequest) { } + @POST + @Path("/search") + @Produces(MediaType.APPLICATION_JSON) + public Uni search(SearchRequest searchRequest) { + + QueryParserRequest queryParserRequest = + getQueryParserRequest(searchRequest); + + Uni queryParserResponseUni = + searcherClient.queryParser(queryParserRequest); + + return queryParserResponseUni + .flatMap(queryParserResponse -> { + + ByteString query = queryParserResponse.getQuery(); + + String searchRequestBody = query.toStringUtf8(); + + ProtocolStringList indexNameList = + queryParserResponse.getIndexNameList(); + + if (indexNameList == null || indexNameList.isEmpty()) { + return Uni.createFrom().item(Response.EMPTY); + } + + String indexNames = + String.join(",", indexNameList); + + var queryParams = queryParserResponse.getQueryParametersMap(); + + org.opensearch.client.Request request = + new org.opensearch.client.Request( + "GET", "/" + indexNames + "/_search"); + + request.addParameters(queryParams); + + request.setJsonEntity(searchRequestBody); + + return Uni.createFrom().emitter((sink) -> restHighLevelClient + .getLowLevelClient() + .performRequestAsync(request, new ResponseListener() { + @Override + public void onSuccess( + org.opensearch.client.Response response) { + try { + SearchResponse searchResponse = + parseEntity( + response.getEntity(), + SearchResponse::fromXContent + ); + + sink.complete(searchResponse); + } + catch (IOException e) { + sink.fail(e); + } + } + + @Override + public void onFailure(Exception e) { + sink.fail(e); + } + })) + .map(this::toSearchResponse); + + }); + + } + @POST @Path("/query-analysis") @Produces(MediaType.APPLICATION_JSON) @@ -137,11 +229,10 @@ public Uni queryAnalysis return searcherClient .queryAnalysis(queryAnalysisRequest) - .map(this::_toQueryAnalysisResponse); + .map(this::toQueryAnalysisResponse); } - @POST @Path("/semantic-autocomplete") @Produces(MediaType.APPLICATION_JSON) @@ -153,123 +244,66 @@ public Uni semanticAutoc return searcherClient .queryAnalysis(queryAnalysisRequest) - .map(this::_toQueryAnalysisResponse); + .map(this::toQueryAnalysisResponse); } - private io.openk9.searcher.queryanalysis.QueryAnalysisResponse _toQueryAnalysisResponse( - QueryAnalysisResponse queryAnalysisResponse) { - - io.openk9.searcher.queryanalysis.QueryAnalysisResponse.QueryAnalysisResponseBuilder - builder = - io.openk9.searcher.queryanalysis.QueryAnalysisResponse.builder(); - - builder.searchText(queryAnalysisResponse.getSearchText()); - - List queryAnalysisTokensList = - new ArrayList<>(); - - for (QueryAnalysisTokens queryAnalysisTokensGRPC : queryAnalysisResponse.getAnalysisList()) { - - io.openk9.searcher.queryanalysis.QueryAnalysisTokens queryAnalysisTokens = - new io.openk9.searcher.queryanalysis.QueryAnalysisTokens(); - - queryAnalysisTokens.setText(queryAnalysisTokensGRPC.getText()); - queryAnalysisTokens.setStart(queryAnalysisTokensGRPC.getStart()); - queryAnalysisTokens.setEnd(queryAnalysisTokensGRPC.getEnd()); - queryAnalysisTokens.setPos(queryAnalysisTokensGRPC.getPosList().toArray(Integer[]::new)); - List tokensList = - queryAnalysisTokensGRPC.getTokensList(); - - Collection> tokens = new ArrayList<>(tokensList.size()); + protected static void mapI18nFields(Map sourceAsMap) { - for (QueryAnalysisSearchToken queryAnalysisSearchToken : tokensList) { + for (Map.Entry entry : sourceAsMap.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Map) { + Map objectMap = (Map) value; + if (objectMap.containsKey("i18n")) { - Map token = new HashMap<>(); + Map i18nMap = + (Map) objectMap.get("i18n"); - token.put("tokenType", queryAnalysisSearchToken.getTokenType()); - token.put("value", queryAnalysisSearchToken.getValue()); - token.put("score", queryAnalysisSearchToken.getScore()); + if (!i18nMap.isEmpty()) { + if (i18nMap.values().iterator().next() instanceof String) { + String i18nString = + (String) i18nMap.values().iterator().next(); + entry.setValue(i18nString); + } + else if (i18nMap.values().iterator().next() instanceof List) { + List i18nList = ((List) i18nMap.values().iterator().next()) + .stream() + .map(object -> String.valueOf(object)) + .toList(); + entry.setValue(i18nList); + } + else { + logger.warn("The object i18nList is not a String or a List"); + } + } - if (StringUtils.isNotBlank(queryAnalysisSearchToken.getKeywordKey())) { - token.put("keywordKey", queryAnalysisSearchToken.getKeywordKey()); } - if (StringUtils.isNotBlank(queryAnalysisSearchToken.getKeywordName())) { - token.put("keywordName", queryAnalysisSearchToken.getKeywordName()); - } - if (StringUtils.isNotBlank(queryAnalysisSearchToken.getEntityType())) { - token.put("entityType", queryAnalysisSearchToken.getEntityType()); - } - if (StringUtils.isNotBlank(queryAnalysisSearchToken.getEntityName())) { - token.put("entityName", queryAnalysisSearchToken.getEntityName()); - } - if (StringUtils.isNotBlank(queryAnalysisSearchToken.getTenantId())) { - token.put("tenantId", queryAnalysisSearchToken.getTenantId()); - } - - if (StringUtils.isNotBlank(queryAnalysisSearchToken.getLabel())) { - token.put("label", queryAnalysisSearchToken.getLabel()); + else if (objectMap.containsKey("base")) { + entry.setValue(objectMap.get("base")); } - - if (!queryAnalysisSearchToken.getExtraMap().isEmpty()) { - token.put("extra", queryAnalysisSearchToken.getExtraMap()); + else { + mapI18nFields((Map) value); } - - tokens.add(token); - } - - queryAnalysisTokens.setTokens(tokens); - - queryAnalysisTokensList.add(queryAnalysisTokens); - - } - - return builder.analysis(queryAnalysisTokensList).build(); - - } - - private QueryAnalysisRequest getQueryAnalysisRequest( - io.openk9.searcher.queryanalysis.QueryAnalysisRequest searchRequest, String mode) { - - QueryAnalysisRequest.Builder builder = - QueryAnalysisRequest.newBuilder(); - - builder.setSearchText(searchRequest.getSearchText()); - builder.setVirtualHost(request.host()); - builder.setJwt(rawToken == null ? "" : rawToken); - builder.setMode(mode); - - if (searchRequest.getTokens() != null) { - - for (QueryAnalysisToken token : searchRequest.getTokens()) { - QueryAnalysisSearchToken.Builder qastBuilder = - _createQastBuilder(token); - builder - .addTokens( - io.openk9.searcher.grpc.QueryAnalysisToken - .newBuilder() - .setText(token.getText()) - .setEnd(token.getEnd()) - .setStart(token.getStart()) - .addAllPos(_toList(token.getPos())) - .setToken(qastBuilder)); + if (value instanceof Iterable) { + for (Object item : (Iterable) value) { + if (item instanceof Map) { + mapI18nFields((Map) item); + } + } } - } - return builder.build(); - } - private static Iterable _toList(Integer[] pos) { + private static Iterable toList(Integer[] pos) { if (pos == null || pos.length == 0) { return List.of(); } return List.of(pos); } - private static QueryAnalysisSearchToken.Builder _createQastBuilder( + private static QueryAnalysisSearchToken.Builder createQastBuilder( QueryAnalysisToken token) { Map tokenMap = token.getToken(); QueryAnalysisSearchToken.Builder qastBuilder = @@ -284,7 +318,7 @@ private static QueryAnalysisSearchToken.Builder _createQastBuilder( switch (key) { case "tokenType" -> qastBuilder.setTokenType(TokenType.valueOf((String) value)); case "value" -> qastBuilder.setValue((String) value); - case "score" -> qastBuilder.setScore(((Number)value).floatValue()); + case "score" -> qastBuilder.setScore(((Number) value).floatValue()); case "keywordKey" -> qastBuilder.setKeywordKey((String) value); case "keywordName" -> qastBuilder.setKeywordName((String) value); case "entityType" -> qastBuilder.setEntityType((String) value); @@ -299,6 +333,12 @@ private static QueryAnalysisSearchToken.Builder _createQastBuilder( private QueryParserRequest getQueryParserRequest(SearchRequest searchRequest) { + var requestBuilder = searcherMapper + .toQueryParserRequest(searchRequest) + .toBuilder(); + + setVectorIndices(searchRequest, requestBuilder); + Map extra = new HashMap<>(); for (String headerName : supportedHeadersName) { @@ -311,9 +351,7 @@ private QueryParserRequest getQueryParserRequest(SearchRequest searchRequest) { String sortAfterKey = searchRequest.getSortAfterKey(); String language = searchRequest.getLanguage(); - return searcherMapper - .toQueryParserRequest(searchRequest) - .toBuilder() + return requestBuilder .setVirtualHost(request.host()) .setJwt(rawToken == null ? "" : rawToken) .putAllExtra(extra) @@ -360,76 +398,165 @@ private Iterable mapToGrpc( } + private static void setVectorIndices( + SearchRequest searchRequest, + QueryParserRequest.Builder requestBuilder) { + var searchTokens = searchRequest.getSearchQuery(); - @POST - @Path("/search") - @Produces(MediaType.APPLICATION_JSON) - public Uni search(SearchRequest searchRequest) { + for (ParserSearchToken token : searchTokens) { - QueryParserRequest queryParserRequest = - getQueryParserRequest(searchRequest); + var tokenType = token.getTokenType(); - Uni queryParserResponseUni = - searcherClient.queryParser(queryParserRequest); + if (tokenType != null + && (tokenType.equalsIgnoreCase("knn") + || tokenType.equalsIgnoreCase("hybrid"))) { - return queryParserResponseUni - .flatMap(queryParserResponse -> { + requestBuilder.setVectorIndices(true); + break; + } - ByteString query = queryParserResponse.getQuery(); + } + } - String searchRequestBody = query.toStringUtf8(); + private static String getHighlightName(String highlightName) { + Matcher matcher = i18nHighlithKeyPattern.matcher(highlightName); + if (matcher.find()) { + return matcher.replaceFirst(""); + } + else { + return highlightName; + } + } - ProtocolStringList indexNameList = - queryParserResponse.getIndexNameList(); + private io.openk9.searcher.queryanalysis.QueryAnalysisResponse toQueryAnalysisResponse( + QueryAnalysisResponse queryAnalysisResponse) { - if (indexNameList == null || indexNameList.isEmpty()) { - return Uni.createFrom().item(Response.EMPTY); + io.openk9.searcher.queryanalysis.QueryAnalysisResponse.QueryAnalysisResponseBuilder + builder = + io.openk9.searcher.queryanalysis.QueryAnalysisResponse.builder(); + + builder.searchText(queryAnalysisResponse.getSearchText()); + + List queryAnalysisTokensList = + new ArrayList<>(); + + for (QueryAnalysisTokens queryAnalysisTokensGRPC : queryAnalysisResponse.getAnalysisList()) { + + io.openk9.searcher.queryanalysis.QueryAnalysisTokens queryAnalysisTokens = + new io.openk9.searcher.queryanalysis.QueryAnalysisTokens(); + + queryAnalysisTokens.setText(queryAnalysisTokensGRPC.getText()); + queryAnalysisTokens.setStart(queryAnalysisTokensGRPC.getStart()); + queryAnalysisTokens.setEnd(queryAnalysisTokensGRPC.getEnd()); + queryAnalysisTokens.setPos(queryAnalysisTokensGRPC.getPosList().toArray(Integer[]::new)); + List tokensList = + queryAnalysisTokensGRPC.getTokensList(); + + Collection> tokens = new ArrayList<>(tokensList.size()); + + for (QueryAnalysisSearchToken queryAnalysisSearchToken : tokensList) { + + Map token = new HashMap<>(); + + token.put("tokenType", queryAnalysisSearchToken.getTokenType()); + token.put("value", queryAnalysisSearchToken.getValue()); + token.put("score", queryAnalysisSearchToken.getScore()); + + if (StringUtils.isNotBlank(queryAnalysisSearchToken.getKeywordKey())) { + token.put("keywordKey", queryAnalysisSearchToken.getKeywordKey()); + } + if (StringUtils.isNotBlank(queryAnalysisSearchToken.getKeywordName())) { + token.put("keywordName", queryAnalysisSearchToken.getKeywordName()); + } + if (StringUtils.isNotBlank(queryAnalysisSearchToken.getEntityType())) { + token.put("entityType", queryAnalysisSearchToken.getEntityType()); + } + if (StringUtils.isNotBlank(queryAnalysisSearchToken.getEntityName())) { + token.put("entityName", queryAnalysisSearchToken.getEntityName()); + } + if (StringUtils.isNotBlank(queryAnalysisSearchToken.getTenantId())) { + token.put("tenantId", queryAnalysisSearchToken.getTenantId()); } - String indexNames = - String.join(",", indexNameList); + if (StringUtils.isNotBlank(queryAnalysisSearchToken.getLabel())) { + token.put("label", queryAnalysisSearchToken.getLabel()); + } - var queryParams = queryParserResponse.getQueryParametersMap(); + if (!queryAnalysisSearchToken.getExtraMap().isEmpty()) { + token.put("extra", queryAnalysisSearchToken.getExtraMap()); + } - org.opensearch.client.Request request = - new org.opensearch.client.Request( - "GET", "/" + indexNames + "/_search"); + tokens.add(token); - request.addParameters(queryParams); + } - request.setJsonEntity(searchRequestBody); + queryAnalysisTokens.setTokens(tokens); - return Uni.createFrom().emitter((sink) -> restHighLevelClient - .getLowLevelClient() - .performRequestAsync(request, new ResponseListener() { - @Override - public void onSuccess( - org.opensearch.client.Response response) { - try { - SearchResponse searchResponse = - parseEntity(response.getEntity(), - SearchResponse::fromXContent); - - sink.complete(searchResponse); - } - catch (IOException e) { - sink.fail(e); - } - } + queryAnalysisTokensList.add(queryAnalysisTokens); - @Override - public void onFailure(Exception e) { - sink.fail(e); - } - })) - .map(this::_toSearchResponse); + } + + return builder.analysis(queryAnalysisTokensList).build(); + + } + + private NamedXContentRegistry getNamedXContentRegistry() { + + return namedXContentRegistryMap.computeIfAbsent( + namedXContentRegistryKey, o -> { + try { + Field registry = + RestHighLevelClient.class.getDeclaredField("registry"); + + registry.setAccessible(true); + return (NamedXContentRegistry) registry.get(restHighLevelClient); + } + catch (Exception e) { + throw new RuntimeException(e); + } }); } - protected final Resp parseEntity(final HttpEntity entity, - final CheckedFunction entityParser) throws IOException { + private QueryAnalysisRequest getQueryAnalysisRequest( + io.openk9.searcher.queryanalysis.QueryAnalysisRequest searchRequest, String mode) { + + QueryAnalysisRequest.Builder builder = + QueryAnalysisRequest.newBuilder(); + + builder.setSearchText(searchRequest.getSearchText()); + builder.setVirtualHost(request.host()); + builder.setJwt(rawToken == null ? "" : rawToken); + builder.setMode(mode); + + if (searchRequest.getTokens() != null) { + + for (QueryAnalysisToken token : searchRequest.getTokens()) { + QueryAnalysisSearchToken.Builder qastBuilder = + createQastBuilder(token); + builder + .addTokens( + io.openk9.searcher.grpc.QueryAnalysisToken + .newBuilder() + .setText(token.getText()) + .setEnd(token.getEnd()) + .setStart(token.getStart()) + .addAllPos(toList(token.getPos())) + .setToken(qastBuilder)); + } + + } + + return builder.build(); + + } + + private Resp parseEntity( + final HttpEntity entity, + final CheckedFunction entityParser) + throws IOException { + if (entity == null) { throw new IllegalStateException("Response body expected but not returned"); } @@ -446,8 +573,8 @@ protected final Resp parseEntity(final HttpEntity entity, } } - private Response _toSearchResponse(SearchResponse searchResponse) { - _printShardFailures(searchResponse); + private Response toSearchResponse(SearchResponse searchResponse) { + printShardFailures(searchResponse); SearchHits hits = searchResponse.getHits(); @@ -508,65 +635,7 @@ private Response _toSearchResponse(SearchResponse searchResponse) { return new Response(result, totalHits.value); } - protected static void mapI18nFields(Map sourceAsMap) { - - for (Map.Entry entry : sourceAsMap.entrySet()) { - Object value = entry.getValue(); - if (value instanceof Map) { - Map objectMap = (Map) value; - if (objectMap.containsKey("i18n")) { - - Map i18nMap = - (Map) objectMap.get("i18n"); - - if (!i18nMap.isEmpty()) { - if (i18nMap.values().iterator().next() instanceof String) { - String i18nString = - (String) i18nMap.values().iterator().next(); - entry.setValue(i18nString); - } - else if (i18nMap.values().iterator().next() instanceof List) { - List i18nList = ((List) i18nMap.values().iterator().next()) - .stream() - .map(object -> String.valueOf(object)) - .toList(); - entry.setValue(i18nList); - } - else { - logger.warn("The object i18nList is not a String or a List"); - } - } - - } - else if (objectMap.containsKey("base")) { - entry.setValue(objectMap.get("base")); - } - else { - mapI18nFields((Map) value); - } - } - if (value instanceof Iterable) { - for (Object item: (Iterable) value) { - if (item instanceof Map) { - mapI18nFields((Map) item); - } - } - } - } - - } - - private static String getHighlightName(String highlightName) { - Matcher matcher = i18nHighlithKeyPattern.matcher(highlightName); - if (matcher.find()) { - return matcher.replaceFirst(""); - } - else { - return highlightName; - } - } - - private void _printShardFailures(SearchResponse searchResponse) { + private void printShardFailures(SearchResponse searchResponse) { if (searchResponse.getShardFailures() != null) { for (ShardSearchFailure failure : searchResponse.getShardFailures()) { logger.warn(failure.reason()); @@ -574,55 +643,4 @@ private void _printShardFailures(SearchResponse searchResponse) { } } - @Inject - SearcherMapper searcherMapper; - - @Inject - InternalSearcherMapper internalSearcherMapper; - - private NamedXContentRegistry getNamedXContentRegistry() { - - return namedXContentRegistryMap.computeIfAbsent( - namedXContentRegistryKey, o -> { - try { - Field registry = - RestHighLevelClient.class.getDeclaredField("registry"); - - registry.setAccessible(true); - - return (NamedXContentRegistry) registry.get(restHighLevelClient); - } - catch (Exception e) { - throw new RuntimeException(e); - } - }); - - } - - @GrpcClient("searcher") - Searcher searcherClient; - - - static Logger logger = Logger.getLogger(SearchResource.class); - - @Inject - @Claim(standard = Claims.raw_token) - String rawToken; - - @Context - HttpServerRequest request; - - @Context - HttpHeaders - headers; - - @ConfigProperty(name = "openk9.searcher.supported.headers.name", defaultValue = "OPENK9_ACL") - List supportedHeadersName; - - private final Map namedXContentRegistryMap = - Collections.synchronizedMap(new IdentityHashMap<>()); - - private static final Object namedXContentRegistryKey = new Object(); - private static final Pattern i18nHighlithKeyPattern = Pattern.compile("\\.i18n\\..{5,}$|\\.base$"); - } diff --git a/js-packages/admin-ui/src/components/EmbeddingModelCreate.tsx b/js-packages/admin-ui/src/components/EmbeddingModelCreate.tsx index c5add9e0e..d094c88d0 100644 --- a/js-packages/admin-ui/src/components/EmbeddingModelCreate.tsx +++ b/js-packages/admin-ui/src/components/EmbeddingModelCreate.tsx @@ -69,8 +69,8 @@ export function EmbeddingModelCreate() { > - - + +
diff --git a/js-packages/admin-ui/src/components/LargeLanguageModelCreate.tsx b/js-packages/admin-ui/src/components/LargeLanguageModelCreate.tsx index cb1d89026..94b3bccd4 100644 --- a/js-packages/admin-ui/src/components/LargeLanguageModelCreate.tsx +++ b/js-packages/admin-ui/src/components/LargeLanguageModelCreate.tsx @@ -83,8 +83,8 @@ export function LargeLanguageModel() { > - - + +
diff --git a/js-packages/search-frontend/src/App.tsx b/js-packages/search-frontend/src/App.tsx index db6b9152e..7c11e0115 100644 --- a/js-packages/search-frontend/src/App.tsx +++ b/js-packages/search-frontend/src/App.tsx @@ -590,7 +590,14 @@ export function App() {
openk9.updateConfiguration({ results: element })} + ref={(element) => + openk9.updateConfiguration({ + resultList: { + element, + changeOnOver: false, + }, + }) + } css={css` grid-area: result; margin-top: ${searchText !== undefined ? "20px" : "unset"}; diff --git a/js-packages/search-frontend/src/components/Result.tsx b/js-packages/search-frontend/src/components/Result.tsx index 91b463d17..cff4c89f0 100644 --- a/js-packages/search-frontend/src/components/Result.tsx +++ b/js-packages/search-frontend/src/components/Result.tsx @@ -7,9 +7,11 @@ import { PdfResult } from "../renderers/openk9/pdf/PdfResult"; import { Renderers } from "./useRenderers"; import { MobileLogoSvg } from "../svgElement/MobileLogoSvg"; import { useTranslation } from "react-i18next"; +import { TemplatesProps } from "../embeddable/entry"; type ResultProps = { renderers: Renderers; + templateCustom?: TemplatesProps | null; result: GenericResultItem; onDetail(result: GenericResultItem | null): void; isMobile: boolean; @@ -22,15 +24,35 @@ type ResultProps = { | undefined | null; }; + function Result(props: ResultProps) { const result = props.result as any; - const { onDetail, renderers } = props; - const isMobile = props.isMobile; - const setIdPreview = props.setIdPreview; - const setDetailMobile = props.setDetailMobile; - const viewButton = props.viewButton; - const overChangeCard = props.overChangeCard; - const setViewButtonDetail = props.setViewButtonDetail; + const { + onDetail, + renderers, + isMobile, + setIdPreview, + setDetailMobile, + viewButton, + overChangeCard, + setViewButtonDetail, + templateCustom, + } = props; + + const classContainer = result?.source?.documentTypes + .map((element: string) => "openk9-card-" + element) + .join(" ", ""); + + const getCustomTemplate = () => { + if (!templateCustom || templateCustom.length === 0) return null; + + const matchedTemplate = templateCustom.find((template) => + result.source.documentTypes.includes(template.source), + ); + + return matchedTemplate?.Template ?? null; + }; + return (
(props: ResultProps) { `} > {(() => { + const CustomTemplate = getCustomTemplate(); + + if (CustomTemplate) { + return ( + + + {isMobile && ( + + )} + + ); + } + const Renderer: React.FC> = result.source.documentTypes .map((k: string) => renderers?.resultRenderers[k]) .find(Boolean); - if (Renderer) { + + if (Renderer && typeof Renderer === "function") { return ( - {isMobile && - CreateButton({ - setIdPreview, - setDetailMobile, - result, - })} - {viewButton && - !isMobile && - ButtonDetail({ - result, - onDetail, - setIdPreview, - setViewButtonDetail, - })} + {isMobile && ( + + )} + {viewButton && !isMobile && ( + + )} ); } + if (result.source.documentTypes.includes("pdf")) { return ( - {isMobile && - CreateButton({ - setIdPreview, - setDetailMobile, - result, - })} - {viewButton && - !isMobile && - ButtonDetail({ - result, - onDetail, - setIdPreview, - setViewButtonDetail, - })} + {isMobile && ( + + )} ); } + if (result.source.documentTypes.includes("document")) { return ( - {isMobile && - CreateButton({ - setIdPreview, - setDetailMobile, - result, - })} - {viewButton && - !isMobile && - ButtonDetail({ - result, - onDetail, - setIdPreview, - setViewButtonDetail, - })} + {isMobile && ( + + )} + {viewButton && !isMobile && ( + + )} ); } + if (result.source.documentTypes.includes("web")) { return ( - {isMobile && - CreateButton({ - setIdPreview, - setDetailMobile, - result, - })} - {viewButton && - !isMobile && - ButtonDetail({ - result, - onDetail, - setIdPreview, - setViewButtonDetail, - })} + {isMobile && ( + + )} + {viewButton && !isMobile && ( + + )} ); } + return ( 
(props: ResultProps) {
); } + function CreateButton({ setDetailMobile, result, @@ -201,7 +242,7 @@ function CreateButton({ setDetailMobile(result); }} > - {t("preview")} + {t("preview")}
@@ -263,7 +304,7 @@ function ButtonDetail({ if (recoveryButton) recoveryButton.focus(); }} > - {t("preview")} + {t("preview")}
diff --git a/js-packages/search-frontend/src/components/ResultList.tsx b/js-packages/search-frontend/src/components/ResultList.tsx index 48f03e3e0..0d2b51966 100644 --- a/js-packages/search-frontend/src/components/ResultList.tsx +++ b/js-packages/search-frontend/src/components/ResultList.tsx @@ -13,6 +13,7 @@ import { ResultSvg } from "../svgElement/ResultSvg"; import { useTranslation } from "react-i18next"; import { result } from "lodash"; import { Options, setSortResultsType } from "./SortResults"; +import { TemplatesProps } from "../embeddable/entry"; const OverlayScrollbarsComponentDockerFix = OverlayScrollbarsComponent as any; // for some reason this component breaks build inside docker export type ResultsDisplayMode = @@ -40,7 +41,9 @@ type ResultsProps = { selectOptions: Options; classNameLabel?: string | undefined; viewButton: boolean; - NoResultsCustom?: React.ReactNode; + templateCustom: TemplatesProps | null; + NoResultsCustom?: any | undefined | null; + BoxTitle?: any | undefined | null; setViewButtonDetail: React.Dispatch>; selectedSort: any; setSelectedSort: setSortResultsType; @@ -74,6 +77,8 @@ function Results({ selectedSort, setSelectedSort, setViewButtonDetail, + BoxTitle, + templateCustom, }: ResultsProps) { const renderers = useRenderers(); @@ -130,6 +135,7 @@ function Results({ setViewButtonDetail={setViewButtonDetail} noResultsCustom={NoResultsCustom} setSelectedSort={setSelectedSort} + templateCustom={templateCustom} /> ); case "virtual": @@ -376,6 +382,7 @@ export function FiniteResults({ > {results.data?.pages[0].total} + {results.data?.pages[0].result.map((result, index) => { return ( @@ -415,6 +422,7 @@ type InfiniteResultsProps = ResulListProps & { viewButton: boolean; noResultsCustom: React.ReactNode; setSelectedSort: setSortResultsType; + templateCustom?: TemplatesProps | null; }; export function InfiniteResults({ renderers, @@ -438,6 +446,7 @@ export function InfiniteResults({ memoryResults, noResultsCustom, viewButton, + templateCustom, setViewButtonDetail, }: InfiniteResultsProps) { const results = useInfiniteResults( @@ -544,6 +553,7 @@ export function InfiniteResults({ setIdPreview={setIdPreview} viewButton={viewButton} setViewButtonDetail={setViewButtonDetail} + templateCustom={templateCustom} /> ); diff --git a/js-packages/search-frontend/src/embeddable/Main.tsx b/js-packages/search-frontend/src/embeddable/Main.tsx index 056b3f4d2..29e717ce8 100644 --- a/js-packages/search-frontend/src/embeddable/Main.tsx +++ b/js-packages/search-frontend/src/embeddable/Main.tsx @@ -246,29 +246,26 @@ export function Main({ )} {renderPortal( - {isSearchLoading ? null : ( - - )} + , configuration.searchConfigurable ? configuration.searchConfigurable.element @@ -515,6 +512,7 @@ export function Main({ classNameLabel={classNameLabelSort} memoryResults={memoryResults} viewButton={viewButton} + templateCustom={configuration.template} setViewButtonDetail={setViewButtonDetail} selectedSort={selectedSort} setSelectedSort={setSelectedSort} @@ -584,6 +582,7 @@ export function Main({ viewButton={viewButton} setViewButtonDetail={setViewButtonDetail} NoResultsCustom={configuration.resultList?.noResultsCustom} + templateCustom={configuration.template} /> )} , diff --git a/js-packages/search-frontend/src/embeddable/entry.tsx b/js-packages/search-frontend/src/embeddable/entry.tsx index 1655c7d14..b0d65f74d 100644 --- a/js-packages/search-frontend/src/embeddable/entry.tsx +++ b/js-packages/search-frontend/src/embeddable/entry.tsx @@ -319,6 +319,8 @@ type searchWithSuggestionsProps = | null | undefined; +export type TemplatesProps = Array<{ source: string; Template: React.FC }>; + export type Configuration = { // simple types debounceTimeSearch: number | null | undefined; @@ -385,6 +387,7 @@ export type Configuration = { sortResultConfigurable: SortResultConfigurableProps | null; sortResultListCustom: SortResultListCustomProps | null; tabsConfigurable: TabsProps | null; + template: TemplatesProps | null; totalResultConfigurable: totalResultProps | null; // functions changeSortResult: ( @@ -445,6 +448,7 @@ const defaultConfiguration: Configuration = { skeletonTabsCustom: null, tabs: null, tabsConfigurable: null, + template: null, tenant: null, token: null, totalResult: null,