Skip to content

Commit

Permalink
Spillende random søk
Browse files Browse the repository at this point in the history
  • Loading branch information
krharum committed Nov 2, 2023
1 parent d754618 commit 8d1144d
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 151 deletions.
2 changes: 1 addition & 1 deletion apps/dolly-backend/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-validation'
implementation 'org.springframework.retry:spring-retry:2.0.0'

implementation 'org.springframework.data:spring-data-elasticsearch'
implementation 'org.springframework.boot:spring-boot-starter-data-elasticsearch'
implementation 'org.opensearch.client:spring-data-opensearch:1.2.0'

implementation 'org.apache.commons:commons-lang3:3.12.0'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,11 @@
import no.nav.dolly.elastic.BestillingElasticRepository;
import no.nav.dolly.elastic.ElasticBestilling;
import no.nav.dolly.elastic.ElasticTyper;
import no.nav.testnav.libs.dto.personsearchservice.v1.search.PersonSearch;
import org.opensearch.client.RestHighLevelClient;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.functionscore.RandomScoreFunctionBuilder;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.Criteria;
import org.springframework.data.elasticsearch.core.query.CriteriaQuery;
import org.springframework.data.elasticsearch.core.query.StringQuery;
import org.springframework.stereotype.Service;

import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.stream.StreamSupport;

@Slf4j
Expand All @@ -33,14 +18,12 @@
public class OpenSearchService {

private final BestillingElasticRepository bestillingElasticRepository;
private final ElasticsearchOperations searchOperations;
private final RestHighLevelClient restHighLevelClient;
private Random random = new SecureRandom();
private final RandomSearchHelperService randomSearchHelperService;

public List<ElasticBestilling> getAll() {

var all = bestillingElasticRepository.findAll();
return StreamSupport.stream(all.spliterator(),false)
return StreamSupport.stream(all.spliterator(), false)
.toList();
}

Expand All @@ -52,89 +35,37 @@ public List<String> getTyper(ElasticTyper[] typer) {
.map(this::getCriteria)
.forEach(criteria::and);

var functionScore = new StringQuery("{\n" +
" \"query\": {\n" +
" \"function_score\": {\n" +
" \"random_score\": {\n" +
" \"seed\": 10,\n" +
" \"field\": \"_seq_no\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}");

var query = new CriteriaQuery(criteria)
.setPageable(Pageable.ofSize(10))
.getRe

// var query = new NativeQuery(new NativeQueryBuilder()
// .withQuery()
// .withMaxResults(10)
// .build());

var functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(new RandomScoreFunctionBuilder().seed(random.nextLong()));

// var query = QueryBuilder.getSearchRequest(QueryBuilder.buildPersonSearchQuery(), 1, 10, 10);

var hits = searchOperations.search(query,
ElasticBestilling.class, IndexCoordinates.of("bestilling"));

// SearchResponse hits = null;
// try {
// hits = restHighLevelClient.search(query, RequestOptions.DEFAULT);
//
// } catch (IOException e) {
// throw new RuntimeException(e);
// }

log.info("hits. {}", hits);
// return null;

return hits.getSearchHits().stream()
.map(SearchHit::getContent)
.map(ElasticBestilling::getIdenter)
.flatMap(Collection::stream)
.distinct()
.toList();
return randomSearchHelperService.search(criteria);
}

private Criteria getCriteria(ElasticTyper type) {

return switch(type) {
case AAREG -> new Criteria("aareg").exists();
case INST -> new Criteria("instdata").exists();
case KRRSTUB -> new Criteria("krrstub").exists();
case SIGRUN_LIGNET -> new Criteria("sigrunInntekt").exists();
case SIGRUN_PENSJONSGIVENDE -> new Criteria("sigrunPensjonsgivende").exists();
case ARENA_BRUKER -> new Criteria("arenaBruker").exists();
case ARENA_AAP -> new Criteria("arenaAap").exists();
case ARENA_AAP115 -> new Criteria("arenaAap115").exists();
case ARENA_DAGP -> new Criteria("arenaDagpenger").exists();
case UDISTUB -> new Criteria("udistub").exists();
case INNTK -> new Criteria("inntektstub").exists();
case PEN_INNTEKT -> new Criteria("penInntekt").exists();
case PEN_TP -> new Criteria("penTp").exists();
case PEN_AP -> new Criteria("penAlderspensjon").exists();
case PEN_UT -> new Criteria("penUforetrygd").exists();
case INNTKMELD -> new Criteria("inntektsmelding").exists();
case BRREGSTUB -> new Criteria("brregstub").exists();
case DOKARKIV -> new Criteria("dokarkiv").exists();
case MEDL -> new Criteria("medl").exists();
case HISTARK -> new Criteria("histark").exists();
case SYKEMELDING -> new Criteria("sykemelding").exists();
case SKJERMING -> new Criteria("skjerming").exists();
case PDLPERSON -> new Criteria("pdldata").exists();
case BANKKONTO -> new Criteria("bankkonto").exists();
case ARBEIDSPLASSENCV -> new Criteria("arbeidsplassenCV").exists();
return switch (type) {
case AAREG -> new Criteria("aareg").exists();
case INST -> new Criteria("instdata").exists();
case KRRSTUB -> new Criteria("krrstub").exists();
case SIGRUN_LIGNET -> new Criteria("sigrunInntekt").exists();
case SIGRUN_PENSJONSGIVENDE -> new Criteria("sigrunPensjonsgivende").exists();
case ARENA_BRUKER -> new Criteria("arenaBruker").exists();
case ARENA_AAP -> new Criteria("arenaAap").exists();
case ARENA_AAP115 -> new Criteria("arenaAap115").exists();
case ARENA_DAGP -> new Criteria("arenaDagpenger").exists();
case UDISTUB -> new Criteria("udistub").exists();
case INNTK -> new Criteria("inntektstub").exists();
case PEN_INNTEKT -> new Criteria("penInntekt").exists();
case PEN_TP -> new Criteria("penTp").exists();
case PEN_AP -> new Criteria("penAlderspensjon").exists();
case PEN_UT -> new Criteria("penUforetrygd").exists();
case INNTKMELD -> new Criteria("inntektsmelding").exists();
case BRREGSTUB -> new Criteria("brregstub").exists();
case DOKARKIV -> new Criteria("dokarkiv").exists();
case MEDL -> new Criteria("medl").exists();
case HISTARK -> new Criteria("histark").exists();
case SYKEMELDING -> new Criteria("sykemelding").exists();
case SKJERMING -> new Criteria("skjerming").exists();
case PDLPERSON -> new Criteria("pdldata").exists();
case BANKKONTO -> new Criteria("bankkonto").exists();
case ARBEIDSPLASSENCV -> new Criteria("arbeidsplassenCV").exists();
};
}

private static void addRandomScoreQuery(BoolQueryBuilder queryBuilder, PersonSearch search) {
Optional.ofNullable(search.getRandomSeed())
.ifPresent(value -> {
if (!value.isEmpty()) {
queryBuilder.must(QueryBuilders.functionScoreQuery(new RandomScoreFunctionBuilder().seed(value)));
}
});
}
}
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package no.nav.dolly.elastic.service;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import no.nav.dolly.elastic.ElasticBestilling;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.Criteria;
import org.springframework.data.elasticsearch.core.query.CriteriaQueryBuilder;
import org.springframework.stereotype.Service;

import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.stream.Collector;
import java.util.stream.Collectors;

@Slf4j
@Service
@RequiredArgsConstructor
public class RandomSearchHelperService {

private static final int PAGE_SIZE = 10;
private static final int FACTOR = 2;

private final ElasticsearchOperations elasticsearchOperations;
private Random random = new SecureRandom();

public List<String> search(Criteria criteria) {

var hits = search(criteria, 0);

if (hits.getTotalHits() > PAGE_SIZE * FACTOR) {
var noOfPages = (int) hits.getTotalHits() / (PAGE_SIZE * FACTOR);
var currentPage = random.nextInt(noOfPages);
log.info("Total hits={}, noOfPages={}, currentPage={}", hits.getTotalHits(), noOfPages, currentPage);
hits = search(criteria, currentPage);
}

var identer = hits.getSearchHits().stream()
.map(SearchHit::getContent)
.map(ElasticBestilling::getIdenter)
.flatMap(Collection::stream)
.distinct()
.collect(toShuffledList());

return identer.subList(0, Math.min(identer.size(), PAGE_SIZE));
}

private static final Collector<?, ?, ?> SHUFFLER = Collectors.collectingAndThen(
Collectors.toCollection(ArrayList::new),
list -> {
Collections.shuffle(list);
return list;
}
);

@SuppressWarnings("unchecked")
private static <T> Collector<T, ?, List<T>> toShuffledList() {
return (Collector<T, ?, List<T>>) SHUFFLER;
}

private SearchHits<ElasticBestilling> search(Criteria criteria, int pageNo) {

return elasticsearchOperations.search(new CriteriaQueryBuilder(criteria)
.withPageable(Pageable.ofSize(PAGE_SIZE * FACTOR).withPage(pageNo))
.build(),
ElasticBestilling.class, IndexCoordinates.of("bestilling"));
}
}

0 comments on commit 8d1144d

Please sign in to comment.