From db39a75e1443bac222b41cf1063072c288e2df58 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 21 Oct 2024 14:23:34 +0200 Subject: [PATCH 1/8] Add regex search for more fields --- kubernetes/loculus/templates/_siloDatabaseConfig.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl index d8c2eef65..f94c5009f 100644 --- a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl +++ b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl @@ -1,10 +1,11 @@ {{- define "loculus.siloDatabaseShared" -}} {{- $type := default "string" .type -}} +{{- $regexSearchable := list "authors" "authorAffiliations" "submissionId" "specimenCollectorSampleId" -}} - type: {{ ($type | eq "timestamp") | ternary "int" (($type | eq "authors") | ternary "string" $type) }} {{- if .generateIndex }} generateIndex: {{ .generateIndex }} {{- end }} - {{- if eq .type "authors" }} + {{- if (in $regexSearchable .name) }} lapisAllowsRegexSearch: true {{- end }} {{- end }} From 2c545a7f8b7a5abfaf1f0a74b4c061613787b10c Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 21 Oct 2024 16:02:52 +0200 Subject: [PATCH 2/8] support enable regexsearch in the helm values --- kubernetes/loculus/templates/_common-metadata.tpl | 4 ++++ kubernetes/loculus/templates/_siloDatabaseConfig.tpl | 2 +- kubernetes/loculus/values.yaml | 3 +++ website/src/types/config.ts | 1 + website/src/utils/search.ts | 2 +- 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index 06ffde10b..23661db6c 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -24,6 +24,7 @@ fields: displayName: Submission ID type: string header: Submission details + enableRegex: true - name: isRevocation type: boolean notSearchable: true @@ -182,6 +183,9 @@ organisms: {{- if .autocomplete }} autocomplete: {{ .autocomplete }} {{- end }} + {{- if .enableRegex }} + regexSearch: {{ .enableRegex }} + {{- end}} {{- if .notSearchable }} notSearchable: {{ .notSearchable }} {{- end }} diff --git a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl index f94c5009f..b66aecba6 100644 --- a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl +++ b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl @@ -5,7 +5,7 @@ {{- if .generateIndex }} generateIndex: {{ .generateIndex }} {{- end }} - {{- if (in $regexSearchable .name) }} + {{- if (eq .enableRegex true) }} lapisAllowsRegexSearch: true {{- end }} {{- end }} diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index ce1fb86b5..36070e641 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -451,10 +451,12 @@ defaultOrganismConfig: &defaultOrganismConfig displayName: Isolate name header: Sample details ingest: ncbiIsolateName + enableRegex: true - name: authors displayName: Authors type: authors header: Authors + enableRegex: true truncateColumnDisplayTo: 15 ingest: ncbiSubmitterNames preprocessing: @@ -465,6 +467,7 @@ defaultOrganismConfig: &defaultOrganismConfig displayName: Author affiliations generateIndex: true autocomplete: true + enableRegex: true truncateColumnDisplayTo: 15 header: Authors ingest: ncbiSubmitterAffiliation diff --git a/website/src/types/config.ts b/website/src/types/config.ts index dc674b18c..3da5bc2f1 100644 --- a/website/src/types/config.ts +++ b/website/src/types/config.ts @@ -39,6 +39,7 @@ export const metadata = z.object({ hideOnSequenceDetailsPage: z.boolean().optional(), header: z.string().optional(), rangeSearch: z.boolean().optional(), + regexSearch: z.boolean().optional(), }); export const inputField = z.object({ diff --git a/website/src/utils/search.ts b/website/src/utils/search.ts index 2cd935232..def24b5fd 100644 --- a/website/src/utils/search.ts +++ b/website/src/utils/search.ts @@ -162,7 +162,7 @@ export const getLapisSearchParameters = ( Object.entries(fieldValues).filter(([, value]) => value !== undefined && value !== ''), ); for (const field of expandedSchema) { - if (field.type === 'authors' && sequenceFilters[field.name] !== undefined) { + if (field.regexSearch === true && sequenceFilters[field.name] !== undefined) { sequenceFilters[field.name.concat('.regex')] = makeCaseInsensitiveLiteralSubstringRegex( sequenceFilters[field.name], ); From 6c8d30e35f176c1d024dd2bcbab7221249660c8e Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 21 Oct 2024 16:11:43 +0200 Subject: [PATCH 3/8] cleanup --- kubernetes/loculus/templates/_siloDatabaseConfig.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl index b66aecba6..e6f12b752 100644 --- a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl +++ b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl @@ -1,6 +1,5 @@ {{- define "loculus.siloDatabaseShared" -}} {{- $type := default "string" .type -}} -{{- $regexSearchable := list "authors" "authorAffiliations" "submissionId" "specimenCollectorSampleId" -}} - type: {{ ($type | eq "timestamp") | ternary "int" (($type | eq "authors") | ternary "string" $type) }} {{- if .generateIndex }} generateIndex: {{ .generateIndex }} From 628cf018fc41c2ab272facb9a7f61ccc8e1280a4 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 23 Oct 2024 11:15:33 +0200 Subject: [PATCH 4/8] Update kubernetes/loculus/templates/_siloDatabaseConfig.tpl Co-authored-by: Theo Sanderson --- kubernetes/loculus/templates/_siloDatabaseConfig.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl index e6f12b752..d5a80bccd 100644 --- a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl +++ b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl @@ -4,7 +4,7 @@ {{- if .generateIndex }} generateIndex: {{ .generateIndex }} {{- end }} - {{- if (eq .enableRegex true) }} + {{- if .enableRegex }} lapisAllowsRegexSearch: true {{- end }} {{- end }} From 9221691427dc403a0a692c4750363b78a60e0c8d Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 23 Oct 2024 16:12:44 +0200 Subject: [PATCH 5/8] rename regex to substring search --- kubernetes/loculus/templates/_common-metadata.tpl | 6 +++--- kubernetes/loculus/templates/_siloDatabaseConfig.tpl | 2 +- kubernetes/loculus/values.yaml | 6 +++--- website/src/types/config.ts | 2 +- website/src/utils/search.ts | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index 23661db6c..d040d8019 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -24,7 +24,7 @@ fields: displayName: Submission ID type: string header: Submission details - enableRegex: true + enableSubstringSearch: true - name: isRevocation type: boolean notSearchable: true @@ -183,8 +183,8 @@ organisms: {{- if .autocomplete }} autocomplete: {{ .autocomplete }} {{- end }} - {{- if .enableRegex }} - regexSearch: {{ .enableRegex }} + {{- if .enableSubstringSearch }} + substringSearch: {{ .enableSubstringSearch }} {{- end}} {{- if .notSearchable }} notSearchable: {{ .notSearchable }} diff --git a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl index d5a80bccd..bc22a36fd 100644 --- a/kubernetes/loculus/templates/_siloDatabaseConfig.tpl +++ b/kubernetes/loculus/templates/_siloDatabaseConfig.tpl @@ -4,7 +4,7 @@ {{- if .generateIndex }} generateIndex: {{ .generateIndex }} {{- end }} - {{- if .enableRegex }} + {{- if .enableSubstringSearch }} lapisAllowsRegexSearch: true {{- end }} {{- end }} diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 36070e641..aec6611da 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -451,12 +451,12 @@ defaultOrganismConfig: &defaultOrganismConfig displayName: Isolate name header: Sample details ingest: ncbiIsolateName - enableRegex: true + enableSubstringSearch: true - name: authors displayName: Authors type: authors header: Authors - enableRegex: true + enableSubstringSearch: true truncateColumnDisplayTo: 15 ingest: ncbiSubmitterNames preprocessing: @@ -467,7 +467,7 @@ defaultOrganismConfig: &defaultOrganismConfig displayName: Author affiliations generateIndex: true autocomplete: true - enableRegex: true + enableSubstringSearch: true truncateColumnDisplayTo: 15 header: Authors ingest: ncbiSubmitterAffiliation diff --git a/website/src/types/config.ts b/website/src/types/config.ts index 3da5bc2f1..a46d794c6 100644 --- a/website/src/types/config.ts +++ b/website/src/types/config.ts @@ -39,7 +39,7 @@ export const metadata = z.object({ hideOnSequenceDetailsPage: z.boolean().optional(), header: z.string().optional(), rangeSearch: z.boolean().optional(), - regexSearch: z.boolean().optional(), + substringSearch: z.boolean().optional(), }); export const inputField = z.object({ diff --git a/website/src/utils/search.ts b/website/src/utils/search.ts index def24b5fd..97288f13d 100644 --- a/website/src/utils/search.ts +++ b/website/src/utils/search.ts @@ -162,7 +162,7 @@ export const getLapisSearchParameters = ( Object.entries(fieldValues).filter(([, value]) => value !== undefined && value !== ''), ); for (const field of expandedSchema) { - if (field.regexSearch === true && sequenceFilters[field.name] !== undefined) { + if (field.substringSearch === true && sequenceFilters[field.name] !== undefined) { sequenceFilters[field.name.concat('.regex')] = makeCaseInsensitiveLiteralSubstringRegex( sequenceFilters[field.name], ); From a7913734834129ab7cb50f3318c8c314c221439f Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 5 Nov 2024 13:35:18 +0100 Subject: [PATCH 6/8] ~ --- kubernetes/loculus/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index aec6611da..633ec0480 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -465,8 +465,6 @@ defaultOrganismConfig: &defaultOrganismConfig authors: authors - name: authorAffiliations displayName: Author affiliations - generateIndex: true - autocomplete: true enableSubstringSearch: true truncateColumnDisplayTo: 15 header: Authors From 2288a0e19c5121a7c037c1e3a8d4c279b2fd5371 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 13 Nov 2024 00:00:00 +0100 Subject: [PATCH 7/8] doc string --- docs/src/content/docs/reference/helm-chart-config.mdx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 768b1586f..e3a58e9ee 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -561,6 +561,15 @@ Each organism object has the following fields: probably `generateIndex` should be true. + + `enableSubstringSearch` + Boolean + + + If true, search results will contain results that contain the given value as a substring. + If false (the default), only exact matches will be returned. + + `initiallyVisible` Boolean From 6a4777796e2fbd34bbf89864b37826601273f0c2 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 13 Nov 2024 00:02:14 +0100 Subject: [PATCH 8/8] Add more docs --- docs/src/content/docs/reference/helm-chart-config.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index e3a58e9ee..ef69c57f7 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -568,6 +568,7 @@ Each organism object has the following fields: If true, search results will contain results that contain the given value as a substring. If false (the default), only exact matches will be returned. + This only works for string fields, and you cannot also enable `autocomplete`.