From 51c8e9ff6336ad155abd8e0b74669c2ad4eb63a4 Mon Sep 17 00:00:00 2001 From: aumetra Date: Mon, 2 Oct 2023 12:45:59 +0200 Subject: [PATCH] Remove `kitsune-search-server` (#359) * Remove kitsune-search-server * fix deprecated warning --- Cargo.lock | 695 +----------------- Cargo.toml | 2 - README.md | 1 - crates/kitsune-core/Cargo.toml | 1 - crates/kitsune-core/src/config.rs | 8 - crates/kitsune-core/src/lib.rs | 17 - crates/kitsune-http-signatures/Cargo.toml | 2 +- crates/kitsune-http-signatures/src/lib.rs | 3 +- crates/kitsune-search/Cargo.toml | 6 - crates/kitsune-search/src/error.rs | 8 - crates/kitsune-search/src/grpc.rs | 172 ----- crates/kitsune-search/src/lib.rs | 6 - docker-compose.yml | 16 - flake.nix | 5 - kitsune-search-server/.env | 8 - kitsune-search-server/.gitignore | 1 - kitsune-search-server/Cargo.toml | 35 - kitsune-search-server/Dockerfile.dev | 19 - kitsune-search-server/Dockerfile.prod | 19 - kitsune-search-server/README.md | 12 - kitsune-search-server/proto/Cargo.toml | 21 - kitsune-search-server/proto/build.rs | 17 - kitsune-search-server/proto/src/lib.rs | 22 - kitsune-search-server/src/config.rs | 42 -- kitsune-search-server/src/grpc/mod.rs | 61 -- .../src/grpc/service/index.rs | 185 ----- kitsune-search-server/src/grpc/service/mod.rs | 8 - .../src/grpc/service/search.rs | 89 --- kitsune-search-server/src/lib.rs | 21 - kitsune-search-server/src/main.rs | 66 -- kitsune-search-server/src/search/mod.rs | 77 -- kitsune-search-server/src/search/schema.rs | 199 ----- kitsune-search-server/src/util.rs | 35 - kitsune-search-server/tests/account.rs | 77 -- kitsune-search-server/tests/common.rs | 40 - kitsune-search-server/tests/post.rs | 106 --- kitsune/Cargo.toml | 1 - proto/search/common.proto | 16 - proto/search/index.proto | 86 --- proto/search/search.proto | 48 -- 40 files changed, 42 insertions(+), 2211 deletions(-) delete mode 100644 crates/kitsune-search/src/grpc.rs delete mode 100644 kitsune-search-server/.env delete mode 100644 kitsune-search-server/.gitignore delete mode 100644 kitsune-search-server/Cargo.toml delete mode 100644 kitsune-search-server/Dockerfile.dev delete mode 100644 kitsune-search-server/Dockerfile.prod delete mode 100644 kitsune-search-server/README.md delete mode 100644 kitsune-search-server/proto/Cargo.toml delete mode 100644 kitsune-search-server/proto/build.rs delete mode 100644 kitsune-search-server/proto/src/lib.rs delete mode 100644 kitsune-search-server/src/config.rs delete mode 100644 kitsune-search-server/src/grpc/mod.rs delete mode 100644 kitsune-search-server/src/grpc/service/index.rs delete mode 100644 kitsune-search-server/src/grpc/service/mod.rs delete mode 100644 kitsune-search-server/src/grpc/service/search.rs delete mode 100644 kitsune-search-server/src/lib.rs delete mode 100644 kitsune-search-server/src/main.rs delete mode 100644 kitsune-search-server/src/search/mod.rs delete mode 100644 kitsune-search-server/src/search/schema.rs delete mode 100644 kitsune-search-server/src/util.rs delete mode 100644 kitsune-search-server/tests/account.rs delete mode 100644 kitsune-search-server/tests/common.rs delete mode 100644 kitsune-search-server/tests/post.rs delete mode 100644 proto/search/common.proto delete mode 100644 proto/search/index.proto delete mode 100644 proto/search/search.proto diff --git a/Cargo.lock b/Cargo.lock index 603b807d6..f14a17d4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,12 +74,6 @@ dependencies = [ "alloc-no-stdlib", ] -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "ammonia" version = "3.3.0" @@ -1059,15 +1053,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bitpacking" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" -dependencies = [ - "crunchy", -] - [[package]] name = "blake2" version = "0.10.6" @@ -1152,9 +1137,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecount" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" +checksum = "ad152d03a2c813c80bb94fedbf3a3f02b28f793e39e7c214c8a0bcc196343de7" [[package]] name = "byteorder" @@ -1243,12 +1228,6 @@ dependencies = [ "libc", ] -[[package]] -name = "census" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fafee10a5dd1cffcb5cc560e0d0df8803d7355a2b12272e3557dee57314cb6e" - [[package]] name = "cfg-if" version = "1.0.0" @@ -1546,7 +1525,7 @@ dependencies = [ "criterion-plot", "futures", "is-terminal", - "itertools 0.10.5", + "itertools", "num-traits 0.2.16", "once_cell", "oorandom", @@ -1566,7 +1545,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.10.5", + "itertools", ] [[package]] @@ -1612,12 +1591,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - [[package]] name = "crypto-bigint" version = "0.5.3" @@ -2005,12 +1978,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "downcast-rs" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" - [[package]] name = "dtoa" version = "1.0.9" @@ -2241,12 +2208,6 @@ dependencies = [ "regex", ] -[[package]] -name = "fastdivide" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04" - [[package]] name = "fastrand" version = "1.9.0" @@ -2278,12 +2239,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flate2" version = "1.0.27" @@ -2318,16 +2273,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs4" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" -dependencies = [ - "rustix", - "windows-sys 0.48.0", -] - [[package]] name = "futf" version = "0.1.5" @@ -2500,19 +2445,6 @@ dependencies = [ "syn 2.0.37", ] -[[package]] -name = "generator" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" -dependencies = [ - "cc", - "libc", - "log", - "rustversion", - "windows", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2640,10 +2572,6 @@ name = "hashbrown" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" -dependencies = [ - "ahash 0.8.3", - "allocator-api2", -] [[package]] name = "headers" @@ -2715,15 +2643,6 @@ dependencies = [ "digest", ] -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "hostname" version = "0.3.1" @@ -2749,12 +2668,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "htmlescape" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" - [[package]] name = "http" version = "0.2.9" @@ -2811,15 +2724,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "human-size" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9994b79e8c1a39b3166c63ae7823bb2b00831e2a96a31399c50fe69df408eaeb" -dependencies = [ - "serde", -] - [[package]] name = "humansize" version = "2.1.3" @@ -2869,18 +2773,6 @@ dependencies = [ "tokio-rustls", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "iana-time-zone" version = "0.1.57" @@ -2991,9 +2883,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", ] [[package]] @@ -3090,15 +2979,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.9" @@ -3130,7 +3010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ "base64 0.21.4", - "ring", + "ring 0.16.20", "serde", "serde_json", ] @@ -3430,7 +3310,7 @@ dependencies = [ "pem", "pkcs8", "rayon", - "ring", + "ring 0.17.0", "thiserror", "time", "tokio", @@ -3495,7 +3375,6 @@ name = "kitsune-search" version = "0.0.1-pre.3" dependencies = [ "async-trait", - "bytes", "diesel", "diesel-async", "diesel_full_text_search", @@ -3503,54 +3382,12 @@ dependencies = [ "futures-util", "kitsune-db", "kitsune-language", - "kitsune-search-proto", "meilisearch-sdk", "serde", "speedy-uuid", "strum", "thiserror", - "tonic", - "tracing", -] - -[[package]] -name = "kitsune-search-proto" -version = "0.0.1-pre.3" -dependencies = [ - "prost", - "protoc-bin-vendored", - "serde", - "tonic", - "tonic-build", -] - -[[package]] -name = "kitsune-search-server" -version = "0.0.1-pre.3" -dependencies = [ - "autometrics", - "dotenvy", - "envy", - "futures-util", - "human-size", - "kitsune-search-proto", - "metrics", - "metrics-exporter-prometheus", - "metrics-tracing-context", - "metrics-util", - "mimalloc", - "prost", - "rand", - "serde", - "tantivy", - "tempfile", - "time", - "tokio", - "tonic", - "tonic-health", - "tower-http", "tracing", - "tracing-subscriber", ] [[package]] @@ -3638,12 +3475,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "levenshtein_automata" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" - [[package]] name = "lexical-core" version = "0.8.5" @@ -3838,35 +3669,6 @@ dependencies = [ "logos-codegen", ] -[[package]] -name = "loom" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" -dependencies = [ - "cfg-if", - "generator", - "pin-utils", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "lru" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" -dependencies = [ - "hashbrown 0.14.1", -] - -[[package]] -name = "lz4_flex" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" - [[package]] name = "mac" version = "0.1.1" @@ -3908,15 +3710,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - [[package]] name = "matchit" version = "0.7.3" @@ -3933,16 +3726,6 @@ dependencies = [ "digest", ] -[[package]] -name = "measure_time" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56220900f1a0923789ecd6bf25fbae8af3b2f1ff3e9e297fc9b6b8674dd4d852" -dependencies = [ - "instant", - "log", -] - [[package]] name = "meilisearch-index-setting-macro" version = "0.24.2" @@ -3984,18 +3767,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" - -[[package]] -name = "memmap2" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" -dependencies = [ - "libc", -] +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -4234,18 +4008,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "murmurhash32" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9380db4c04d219ac5c51d14996bbf2c2e9a15229771b53f8671eb6c83cf44df" - [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -4402,15 +4164,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" -[[package]] -name = "oneshot" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f6640c6bda7731b1fdbab747981a0f896dd1fedaf9f4a53fa237a04a84431f4" -dependencies = [ - "loom", -] - [[package]] name = "oorandom" version = "11.1.3" @@ -4428,7 +4181,7 @@ dependencies = [ "dyn-clone", "hmac", "http", - "itertools 0.10.5", + "itertools", "log", "oauth2", "p256", @@ -4507,15 +4260,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "ownedbytes" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "owo-colors" version = "3.5.0" @@ -4672,16 +4416,6 @@ dependencies = [ "ucd-trie", ] -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.0.2", -] - [[package]] name = "phf" version = "0.10.1" @@ -4932,16 +4666,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prettyplease" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" -dependencies = [ - "proc-macro2", - "syn 2.0.37", -] - [[package]] name = "primeorder" version = "0.13.2" @@ -5000,110 +4724,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" -dependencies = [ - "bytes", - "heck", - "itertools 0.11.0", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.37", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" -dependencies = [ - "anyhow", - "itertools 0.11.0", - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "prost-types" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" -dependencies = [ - "prost", -] - -[[package]] -name = "protoc-bin-vendored" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d" -dependencies = [ - "protoc-bin-vendored-linux-aarch_64", - "protoc-bin-vendored-linux-ppcle_64", - "protoc-bin-vendored-linux-x86_32", - "protoc-bin-vendored-linux-x86_64", - "protoc-bin-vendored-macos-x86_64", - "protoc-bin-vendored-win32", -] - -[[package]] -name = "protoc-bin-vendored-linux-aarch_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435" - -[[package]] -name = "protoc-bin-vendored-linux-ppcle_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516" - -[[package]] -name = "protoc-bin-vendored-linux-x86_32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0" - -[[package]] -name = "protoc-bin-vendored-linux-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924" - -[[package]] -name = "protoc-bin-vendored-macos-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537" - -[[package]] -name = "protoc-bin-vendored-win32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" - [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -5286,19 +4906,10 @@ checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", + "regex-automata", "regex-syntax 0.7.5", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - [[package]] name = "regex-automata" version = "0.3.9" @@ -5359,11 +4970,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb9d44f9bf6b635117787f72416783eb7e4227aaf255e5ce739563d817176a7e" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rmp" version = "0.8.12" @@ -5455,16 +5080,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -5506,7 +5121,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", - "ring", + "ring 0.16.20", "rustls-webpki 0.101.6", "sct", ] @@ -5538,8 +5153,8 @@ version = "0.100.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -5548,8 +5163,8 @@ version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -5592,12 +5207,6 @@ dependencies = [ "pin-utils", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -5626,8 +5235,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -5981,9 +5590,6 @@ name = "sketches-ddsketch" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" -dependencies = [ - "serde", -] [[package]] name = "slab" @@ -6213,146 +5819,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "tantivy" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1d4675fed6fe2218ce11445374e181e864a8ffd0f28e7e0591ccfc38cd000ae" -dependencies = [ - "aho-corasick", - "arc-swap", - "async-trait", - "base64 0.21.4", - "bitpacking", - "byteorder", - "census", - "crc32fast", - "crossbeam-channel", - "downcast-rs", - "fastdivide", - "fs4", - "htmlescape", - "itertools 0.11.0", - "levenshtein_automata", - "log", - "lru", - "lz4_flex", - "measure_time", - "memmap2", - "murmurhash32", - "num_cpus", - "once_cell", - "oneshot", - "rayon", - "regex", - "rust-stemmers", - "rustc-hash", - "serde", - "serde_json", - "sketches-ddsketch", - "smallvec", - "tantivy-bitpacker", - "tantivy-columnar", - "tantivy-common", - "tantivy-fst", - "tantivy-query-grammar", - "tantivy-stacker", - "tantivy-tokenizer-api", - "tempfile", - "thiserror", - "time", - "uuid", - "winapi", -] - -[[package]] -name = "tantivy-bitpacker" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a" -dependencies = [ - "bitpacking", -] - -[[package]] -name = "tantivy-columnar" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70" -dependencies = [ - "fastdivide", - "fnv", - "itertools 0.11.0", - "serde", - "tantivy-bitpacker", - "tantivy-common", - "tantivy-sstable", - "tantivy-stacker", -] - -[[package]] -name = "tantivy-common" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1" -dependencies = [ - "async-trait", - "byteorder", - "ownedbytes", - "serde", - "time", -] - -[[package]] -name = "tantivy-fst" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944" -dependencies = [ - "byteorder", - "regex-syntax 0.6.29", - "utf8-ranges", -] - -[[package]] -name = "tantivy-query-grammar" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77" -dependencies = [ - "nom", -] - -[[package]] -name = "tantivy-sstable" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df" -dependencies = [ - "tantivy-common", - "tantivy-fst", - "zstd", -] - -[[package]] -name = "tantivy-stacker" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c" -dependencies = [ - "murmurhash32", - "tantivy-common", -] - -[[package]] -name = "tantivy-tokenizer-api" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66" -dependencies = [ - "serde", -] - [[package]] name = "tempfile" version = "3.8.0" @@ -6490,16 +5956,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.1.0" @@ -6656,59 +6112,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tonic" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.21.4", - "bytes", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "tonic-health" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f80db390246dfb46553481f6024f0082ba00178ea495dbb99e70ba9a4fafb5e1" -dependencies = [ - "async-stream", - "prost", - "tokio", - "tokio-stream", - "tonic", -] - [[package]] name = "tower" version = "0.4.13" @@ -6717,13 +6120,9 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", - "slab", "tokio", - "tokio-util", "tower-layer", "tower-service", "tracing", @@ -6842,14 +6241,10 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ - "matchers", "nu-ansi-term", - "once_cell", - "regex", "sharded-slab", "smallvec", "thread_local", - "tracing", "tracing-core", "tracing-log", ] @@ -6965,6 +6360,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.4.1" @@ -6983,12 +6384,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf8-ranges" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" - [[package]] name = "utf8parse" version = "0.2.1" @@ -7233,18 +6628,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "whichlang" version = "0.1.0" @@ -7528,7 +6911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "103fa851fff70ea29af380e87c25c48ff7faac5c530c70bd0e65366d4e0c94e4" dependencies = [ "fancy-regex", - "itertools 0.10.5", + "itertools", "js-sys", "lazy_static", "quick-error", diff --git a/Cargo.toml b/Cargo.toml index 4471e0487..fa537bd3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,8 +30,6 @@ members = [ "kitsune", "kitsune-cli", "kitsune-job-runner", - "kitsune-search-server", - "kitsune-search-server/proto", "lib/athena", "lib/post-process", "lib/speedy-uuid", diff --git a/README.md b/README.md index 28e6606b2..29304f6c5 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ So, as long as this disclaimer is here, make sure to double check all the change - `kitsune-cli/`: Kitsune CLI binary - `kitsune-fe/`: Kitsune frontend project - `kitsune-job-runner/`: Kitsune dedicated job runner -- `kitsune-search-server/`: Search server for Kitsune - `lib/`: Libraries made for Kitsune but with no dependencies on Kitsune-specific code. Easily usable by other projects - `proto/`: Any Protobuf definitions for the Kitsune project - `public/`: Public web assets diff --git a/crates/kitsune-core/Cargo.toml b/crates/kitsune-core/Cargo.toml index c41182d49..ff79bd8de 100644 --- a/crates/kitsune-core/Cargo.toml +++ b/crates/kitsune-core/Cargo.toml @@ -77,7 +77,6 @@ zxcvbn = { version = "2.2.2", default-features = false } [features] default = [] -kitsune-search = ["kitsune-search/kitsune-search"] mastodon-api = [] meilisearch = ["kitsune-search/meilisearch"] diff --git a/crates/kitsune-core/src/config.rs b/crates/kitsune-core/src/config.rs index d131391f3..b5f7eec36 100644 --- a/crates/kitsune-core/src/config.rs +++ b/crates/kitsune-core/src/config.rs @@ -110,13 +110,6 @@ pub enum MessagingConfiguration { InProcess, } -#[derive(Clone, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct KitsuneSearchConfiguration { - pub index_server: SmolStr, - pub search_servers: Vec, -} - #[derive(Clone, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct MeiliSearchConfiguration { @@ -127,7 +120,6 @@ pub struct MeiliSearchConfiguration { #[derive(Clone, Deserialize, Serialize)] #[serde(rename_all = "kebab-case", tag = "type")] pub enum SearchConfiguration { - Kitsune(KitsuneSearchConfiguration), Meilisearch(MeiliSearchConfiguration), Sql, None, diff --git a/crates/kitsune-core/src/lib.rs b/crates/kitsune-core/src/lib.rs index 55073dddd..d1f4113ca 100644 --- a/crates/kitsune-core/src/lib.rs +++ b/crates/kitsune-core/src/lib.rs @@ -69,9 +69,6 @@ use std::{ time::Duration, }; -#[cfg(feature = "kitsune-search")] -use kitsune_search::GrpcSearchService; - #[cfg(feature = "meilisearch")] use kitsune_search::MeiliSearchService; @@ -188,20 +185,6 @@ async fn prepare_search( db_pool: &PgPool, ) -> eyre::Result { let service = match search_config { - SearchConfiguration::Kitsune(_config) => { - #[cfg(not(feature = "kitsune-search"))] - panic!("Server compiled without Kitsune Search compatibility"); - - #[cfg(feature = "kitsune-search")] - #[allow(clippy::used_underscore_binding)] - GrpcSearchService::connect( - &_config.index_server, - _config.search_servers.iter().map(ToString::to_string), - ) - .await - .context("Failed to connect to the search servers")? - .into() - } SearchConfiguration::Meilisearch(_config) => { #[cfg(not(feature = "meilisearch"))] panic!("Server compiled without Meilisearch compatibility"); diff --git a/crates/kitsune-http-signatures/Cargo.toml b/crates/kitsune-http-signatures/Cargo.toml index 6c2220278..beffc93e6 100644 --- a/crates/kitsune-http-signatures/Cargo.toml +++ b/crates/kitsune-http-signatures/Cargo.toml @@ -8,7 +8,7 @@ base64-simd = "0.8.0" derive_builder = "0.12.0" http = "0.2.9" rayon = "1.8.0" -ring = { version = "0.16.20", features = ["std"] } +ring = { version = "0.17.0", features = ["std"] } time = { version = "0.3.29", features = ["formatting", "parsing"] } thiserror = "1.0.49" tokio = { version = "1.32.0", features = ["sync"] } diff --git a/crates/kitsune-http-signatures/src/lib.rs b/crates/kitsune-http-signatures/src/lib.rs index 622e4a52b..27247938f 100644 --- a/crates/kitsune-http-signatures/src/lib.rs +++ b/crates/kitsune-http-signatures/src/lib.rs @@ -98,9 +98,10 @@ impl SigningKey for Ed25519KeyPair { impl SigningKey for RsaKeyPair { fn sign(&self, msg: &[u8]) -> Vec { - let mut signature = vec![0; self.public_modulus_len()]; + let mut signature = vec![0; self.public().modulus_len()]; self.sign(&RSA_PKCS1_SHA256, &SystemRandom::new(), msg, &mut signature) .unwrap(); + signature } } diff --git a/crates/kitsune-search/Cargo.toml b/crates/kitsune-search/Cargo.toml index 5034291d8..d58690246 100644 --- a/crates/kitsune-search/Cargo.toml +++ b/crates/kitsune-search/Cargo.toml @@ -18,15 +18,9 @@ strum = { version = "0.25.0", features = ["derive"] } thiserror = "1.0.49" tracing = "0.1.37" -# "kitsune-search" feature -bytes = { version = "1.5.0", optional = true } -kitsune-search-proto = { path = "../../kitsune-search-server/proto", optional = true } -tonic = { version = "0.10.2", optional = true } - # "meilisearch" feature meilisearch-sdk = { version = "0.24.2", optional = true } [features] default = [] -kitsune-search = ["dep:bytes", "dep:kitsune-search-proto", "dep:tonic"] meilisearch = ["dep:meilisearch-sdk"] diff --git a/crates/kitsune-search/src/error.rs b/crates/kitsune-search/src/error.rs index af8d1deed..c726ddb24 100644 --- a/crates/kitsune-search/src/error.rs +++ b/crates/kitsune-search/src/error.rs @@ -11,14 +11,6 @@ pub enum Error { #[cfg(feature = "meilisearch")] #[error(transparent)] Meilisearch(#[from] meilisearch_sdk::errors::Error), - - #[cfg(feature = "kitsune-search")] - #[error(transparent)] - TonicStatus(#[from] tonic::Status), - - #[cfg(feature = "kitsune-search")] - #[error(transparent)] - TonicTransport(#[from] tonic::transport::Error), } impl From> for Error diff --git a/crates/kitsune-search/src/grpc.rs b/crates/kitsune-search/src/grpc.rs deleted file mode 100644 index 01a8a2d30..000000000 --- a/crates/kitsune-search/src/grpc.rs +++ /dev/null @@ -1,172 +0,0 @@ -use super::{Result, SearchBackend, SearchIndex, SearchItem, SearchResult}; -use async_trait::async_trait; -use bytes::Bytes; -use futures_util::stream; -use kitsune_search_proto::{ - common::SearchIndex as GrpcSearchIndex, - index::{ - add_index_request::IndexEntity, index_client::IndexClient, AddAccountIndex, - AddIndexRequest, AddPostIndex, RemoveIndexRequest, ResetRequest, - }, - search::{search_client::SearchClient, SearchRequest, SearchResult as GrpcSearchResult}, -}; -use speedy_uuid::Uuid; -use std::future; -use tonic::transport::{Channel, Endpoint}; - -/// Search service -/// -/// Connects to the `kitsune-search` backend via gRPC -#[derive(Clone)] -pub struct SearchService { - searcher: SearchClient, - indexer: IndexClient, -} - -impl SearchService { - /// Connect to the search backend via gRPC - /// - /// # Errors - /// - /// - The servers aren't reachable - pub async fn connect(index_endpoint: &str, search_endpoints: I) -> Result - where - I: Iterator + Clone, - T: Into, - { - let index_channel = Endpoint::from_shared(index_endpoint.to_string())? - .connect() - .await?; - - let search_endpoints: Vec = search_endpoints - .clone() - .map(Endpoint::from_shared) - .collect::>()?; - let search_channel = Channel::balance_list(search_endpoints.into_iter()); - - Ok(Self { - searcher: SearchClient::new(search_channel), - indexer: IndexClient::new(index_channel), - }) - } -} - -#[async_trait] -impl SearchBackend for SearchService { - #[instrument(skip_all)] - async fn add_to_index(&self, item: SearchItem) -> Result<()> { - let request = match item { - SearchItem::Account(account) => AddIndexRequest { - index_entity: Some(IndexEntity::Account(AddAccountIndex { - id: account.id.as_bytes().to_vec(), - display_name: account.display_name, - username: account.username, - description: account.note, - })), - }, - SearchItem::Post(post) => AddIndexRequest { - index_entity: Some(IndexEntity::Post(AddPostIndex { - id: post.id.as_bytes().to_vec(), - subject: post.subject, - content: post.content, - })), - }, - }; - - self.indexer - .clone() - .add(stream::once(future::ready(request))) - .await?; - - Ok(()) - } - - #[instrument(skip_all)] - async fn remove_from_index(&self, item: &SearchItem) -> Result<()> { - let request = match item { - SearchItem::Account(account) => RemoveIndexRequest { - index: GrpcSearchIndex::from(SearchIndex::Account).into(), - id: account.id.as_bytes().to_vec(), - }, - SearchItem::Post(post) => RemoveIndexRequest { - index: GrpcSearchIndex::from(SearchIndex::Post).into(), - id: post.id.as_bytes().to_vec(), - }, - }; - - self.indexer - .clone() - .remove(stream::once(future::ready(request))) - .await?; - - Ok(()) - } - - #[instrument(skip(self))] - async fn reset_index(&self, index: SearchIndex) -> Result<()> { - let request = ResetRequest { - index: GrpcSearchIndex::from(index).into(), - }; - self.indexer.clone().reset(request).await?; - - Ok(()) - } - - #[instrument(skip(self))] - async fn search( - &self, - index: SearchIndex, - query: String, - max_results: u64, - offset: u64, - min_id: Option, - max_id: Option, - ) -> Result> { - let request = SearchRequest { - index: GrpcSearchIndex::from(index).into(), - query, - max_results, - offset, - max_id: min_id.as_ref().map(|id| id.as_bytes().to_vec()), - min_id: max_id.as_ref().map(|id| id.as_bytes().to_vec()), - }; - - let results = self - .searcher - .clone() - .search(request) - .await? - .into_inner() - .results - .into_iter() - .map(Into::into) - .collect(); - - Ok(results) - } - - #[instrument(skip_all)] - async fn update_in_index(&self, item: SearchItem) -> Result<()> { - self.remove_from_index(&item).await?; - self.add_to_index(item).await?; - - Ok(()) - } -} - -impl From for GrpcSearchIndex { - fn from(value: SearchIndex) -> Self { - match value { - SearchIndex::Account => GrpcSearchIndex::Account, - SearchIndex::Post => GrpcSearchIndex::Post, - } - } -} - -impl From for SearchResult { - fn from(value: GrpcSearchResult) -> Self { - let id = Uuid::from_slice(&value.id).expect("Received non-UUID from search service"); - - Self { id } - } -} diff --git a/crates/kitsune-search/src/lib.rs b/crates/kitsune-search/src/lib.rs index 967fde434..2ebf84561 100644 --- a/crates/kitsune-search/src/lib.rs +++ b/crates/kitsune-search/src/lib.rs @@ -13,15 +13,11 @@ use speedy_uuid::Uuid; use strum::EnumIter; mod error; -#[cfg(feature = "kitsune-search")] -mod grpc; #[cfg(feature = "meilisearch")] mod meilisearch; mod sql; pub use self::error::Error; -#[cfg(feature = "kitsune-search")] -pub use self::grpc::SearchService as GrpcSearchService; #[cfg(feature = "meilisearch")] pub use self::meilisearch::MeiliSearchService; pub use self::sql::SearchService as SqlSearchService; @@ -31,8 +27,6 @@ type Result = std::result::Result; #[derive(Clone)] #[enum_dispatch(SearchBackend)] pub enum SearchService { - #[cfg(feature = "kitsune-search")] - Grpc(GrpcSearchService), #[cfg(feature = "meilisearch")] Meilisearch(MeiliSearchService), Noop(NoopSearchService), diff --git a/docker-compose.yml b/docker-compose.yml index e23ef6b8f..e2c9e7797 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,8 +16,6 @@ services: JOB_WORKERS: 1 REDIS_URL: redis://redis:6379 RUST_LOG: debug - SEARCH_INDEX_SERVER: http://backend-search:8080 - SEARCH_SERVERS: http://backend-search:8080 volumes: - upload-data:/app/uploads - type: bind @@ -28,20 +26,6 @@ services: - db - redis - backend-search: - image: kitsune-search - networks: - - backend - environment: - INDEX_DIR_PATH: search-index - LEVENSHTEIN_DISTANCE: 2 - MEMORY_ARENA_SIZE: 3MB - PORT: 8080 - READ_ONLY: false - RUST_LOG: debug - volumes: - - search-index:/app/search-index - db: image: postgres:15-alpine environment: diff --git a/flake.nix b/flake.nix index 180b44279..715ba52d2 100644 --- a/flake.nix +++ b/flake.nix @@ -96,11 +96,6 @@ cargoBuildFlags = "-p kitsune"; }); - search = rustPlatform.buildRustPackage (basePackage // { - pname = "kitsune-search"; - cargoBuildFlags = "-p kitsune-search"; - }); - frontend = pkgs.mkYarnPackage { inherit version; diff --git a/kitsune-search-server/.env b/kitsune-search-server/.env deleted file mode 100644 index 2e634c4d3..000000000 --- a/kitsune-search-server/.env +++ /dev/null @@ -1,8 +0,0 @@ -INDEX_DIR_PATH=search-index -LEVENSHTEIN_DISTANCE=2 -MEMORY_ARENA_SIZE=3MB -PORT=8081 -PROMETHEUS_PORT=9001 -READ_ONLY=false - -RUST_LOG=debug diff --git a/kitsune-search-server/.gitignore b/kitsune-search-server/.gitignore deleted file mode 100644 index 366403c27..000000000 --- a/kitsune-search-server/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/search-index diff --git a/kitsune-search-server/Cargo.toml b/kitsune-search-server/Cargo.toml deleted file mode 100644 index 7be0c0f10..000000000 --- a/kitsune-search-server/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "kitsune-search-server" -version.workspace = true -edition.workspace = true - -[dependencies] -autometrics = { version = "0.6.0", default-features = false, features = [ - "metrics", -] } -dotenvy = "0.15.7" -envy = "0.4.2" -futures-util = "0.3.28" -human-size = { version = "0.4.3", features = ["serde"] } -kitsune-search-proto = { path = "proto" } -metrics = "0.21.1" -metrics-exporter-prometheus = "0.12.1" -metrics-tracing-context = "0.14.0" -metrics-util = "0.15.1" -mimalloc = "0.1.39" -prost = "0.12.1" -serde = { version = "1.0.188", features = ["derive"] } -tantivy = "0.21.0" -time = "0.3.29" -tokio = { version = "1.32.0", features = ["full"] } -tonic = "0.10.2" -tonic-health = "0.10.2" -tower-http = { version = "0.4.4", features = ["add-extension", "trace"] } -tracing = "0.1.37" -tracing-subscriber = "0.3.17" - -[features] - -[dev-dependencies] -rand = "0.8.5" -tempfile = "3.8.0" diff --git a/kitsune-search-server/Dockerfile.dev b/kitsune-search-server/Dockerfile.dev deleted file mode 100644 index 4ae8123a1..000000000 --- a/kitsune-search-server/Dockerfile.dev +++ /dev/null @@ -1,19 +0,0 @@ -FROM rust:1-alpine AS base -RUN apk add --no-cache musl-dev make protobuf-dev -RUN cargo install cargo-chef -WORKDIR app - -FROM base AS planner -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - -FROM base AS build -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --recipe-path recipe.json -COPY . . -RUN cargo build --bin kitsune-search-server - -FROM alpine:latest -WORKDIR app -COPY --from=build /app/target/debug/kitsune-search-server . -ENTRYPOINT ["./kitsune-search-server"] diff --git a/kitsune-search-server/Dockerfile.prod b/kitsune-search-server/Dockerfile.prod deleted file mode 100644 index 53214b531..000000000 --- a/kitsune-search-server/Dockerfile.prod +++ /dev/null @@ -1,19 +0,0 @@ -FROM rust:1-alpine AS base -RUN apk add --no-cache musl-dev make protobuf-dev -RUN cargo install cargo-chef -WORKDIR app - -FROM base AS planner -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - -FROM base AS build -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json -COPY . . -RUN cargo build --release --bin kitsune-search-server - -FROM alpine:latest -WORKDIR app -COPY --from=build /app/target/release/kitsune-search-server . -ENTRYPOINT ["./kitsune-search-server"] \ No newline at end of file diff --git a/kitsune-search-server/README.md b/kitsune-search-server/README.md deleted file mode 100644 index 4744e3af2..000000000 --- a/kitsune-search-server/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# kitsune-search - -Tailormade search solution for Kitsune based on the tantivy search engine. -It supports running multiple nodes in parallel using a primary/secondary architecture. - -## Configuration - -- `INDEX_DIR_PATH`: Path to the directory in which the search indices are created. If you are in a container, make sure this is a persistent volume -- `LEVENSHTEIN_DISTANCE`: Configures how lenient the search engine is when determining the whether the query is related to the actual text -- `MEMORY_ARENA_SIZE`: Configures a RAM budget for the search engine. If the budget is exhausted, the documents are flushed to disk (has to be at least 3MB) -- `PORT`: Port on which the gRPC server will listen -- `READ_ONLY`: This node is a *read-only* node; It can only search through the index. diff --git a/kitsune-search-server/proto/Cargo.toml b/kitsune-search-server/proto/Cargo.toml deleted file mode 100644 index 1c9e06825..000000000 --- a/kitsune-search-server/proto/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "kitsune-search-proto" -version.workspace = true -edition.workspace = true - -[dependencies] -prost = "0.12.1" -serde = { version = "1.0.188", features = ["derive"] } -tonic = { version = "0.10.2", default-features = false, features = [ - "codegen", - "transport", - "prost", -] } - -[build-dependencies] -protoc-bin-vendored = { version = "3.0.0", optional = true } -tonic-build = "0.10.2" - -[features] -default = [] -vendored-protoc = ["protoc-bin-vendored"] diff --git a/kitsune-search-server/proto/build.rs b/kitsune-search-server/proto/build.rs deleted file mode 100644 index f8cb13ea4..000000000 --- a/kitsune-search-server/proto/build.rs +++ /dev/null @@ -1,17 +0,0 @@ -fn main() { - #[cfg(feature = "vendored-protoc")] - std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); - - tonic_build::configure() - .protoc_arg("--experimental_allow_proto3_optional") - .type_attribute(".", "#[derive(::serde::Serialize)]") - .compile( - &[ - "../../proto/search/common.proto", - "../../proto/search/index.proto", - "../../proto/search/search.proto", - ], - &["../../proto/search"], - ) - .unwrap(); -} diff --git a/kitsune-search-server/proto/src/lib.rs b/kitsune-search-server/proto/src/lib.rs deleted file mode 100644 index 458dd78a8..000000000 --- a/kitsune-search-server/proto/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! -//! Tonic types generated from protobuf definitions by `tonic-build` -//! - -#![forbid(rust_2018_idioms, unsafe_code)] -#![deny(missing_docs)] -#![allow(forbidden_lint_groups)] - -/// Common types -pub mod common { - tonic::include_proto!("kitsune.common"); -} - -/// Indexing types and services -pub mod index { - tonic::include_proto!("kitsune.index"); -} - -/// Search types and services -pub mod search { - tonic::include_proto!("kitsune.search"); -} diff --git a/kitsune-search-server/src/config.rs b/kitsune-search-server/src/config.rs deleted file mode 100644 index b68ce49f5..000000000 --- a/kitsune-search-server/src/config.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! -//! Service configuration -//! - -use human_size::Size; -use serde::Deserialize; -use std::path::PathBuf; - -fn default_prometheus_port() -> u16 { - 9000 -} - -/// Configuration values -#[derive(Clone, Deserialize)] -pub struct Configuration { - /// Path to the directory in which the indices are created - pub index_dir_path: PathBuf, - - /// Levenshtein distance used when executing the fuzzy search - /// - /// It basically tells the search engine how lenient it should be with matching queries - pub levenshtein_distance: u8, - - /// Size of the memory arena allocated by the writers - /// - /// Every time a writer fills its memory arena, the operations are flushed to disk - pub memory_arena_size: Size, - - /// Port on which the gRPC server is listening on - pub port: u16, - - /// Port on which the Prometheus exporter is listening on - /// - /// Defaults to 9000 - #[serde(default = "default_prometheus_port")] - pub prometheus_port: u16, - - /// Run this node in read-only mode - /// - /// Every index can only have one writer (indexer), so set this to true on all secondary nodes - pub read_only: bool, -} diff --git a/kitsune-search-server/src/grpc/mod.rs b/kitsune-search-server/src/grpc/mod.rs deleted file mode 100644 index b8fa4a038..000000000 --- a/kitsune-search-server/src/grpc/mod.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! -//! gRPC service -//! - -use self::service::{IndexService, SearchService}; -use crate::{config::Configuration, search::SearchIndex}; -use kitsune_search_proto::{index::index_server::IndexServer, search::search_server::SearchServer}; -use tokio::sync::RwLock; -use tonic::transport::Server; -use tower_http::{add_extension::AddExtensionLayer, trace::TraceLayer}; - -pub mod service; - -/// Start the search (and possibly index) gRPC service -#[instrument(skip_all)] -pub async fn start(config: Configuration, search_index: SearchIndex) { - let (mut health_reporter, health_service) = tonic_health::server::health_reporter(); - health_reporter - .set_serving::>() - .await; - health_reporter - .set_serving::>() - .await; - - let account_reader = search_index.indices.account.reader().unwrap(); - let post_reader = search_index.indices.post.reader().unwrap(); - - let mut server = Server::builder() - .layer(AddExtensionLayer::new(config.clone())) - .layer(AddExtensionLayer::new(search_index.clone())) - .layer(TraceLayer::new_for_grpc()) - .add_service(health_service); - - if !config.read_only { - let account_writer = search_index - .indices - .account - .writer(config.memory_arena_size.to_bytes() as usize) - .unwrap(); - - let post_writer = search_index - .indices - .post - .writer(config.memory_arena_size.to_bytes() as usize) - .unwrap(); - - server = server.add_service(IndexServer::new(IndexService { - account: RwLock::new(account_writer), - post: RwLock::new(post_writer), - })); - } - - server - .add_service(SearchServer::new(SearchService { - account: account_reader, - post: post_reader, - })) - .serve(([0, 0, 0, 0], config.port).into()) - .await - .unwrap(); -} diff --git a/kitsune-search-server/src/grpc/service/index.rs b/kitsune-search-server/src/grpc/service/index.rs deleted file mode 100644 index bb1670a1d..000000000 --- a/kitsune-search-server/src/grpc/service/index.rs +++ /dev/null @@ -1,185 +0,0 @@ -use crate::search::SearchIndex; -use autometrics::autometrics; -use futures_util::TryStreamExt; -use kitsune_search_proto::{ - common::SearchIndex as GrpcSearchIndex, - index::{ - add_index_request::IndexEntity, index_server::Index, AddIndexRequest, AddIndexResponse, - RemoveIndexRequest, RemoveIndexResponse, ResetRequest, ResetResponse, - }, -}; -use tantivy::{DateTime, Document, IndexWriter, Term}; -use time::OffsetDateTime; -use tokio::sync::RwLock; -use tonic::{async_trait, Request, Response, Status, Streaming}; - -/// Maximum amount of documents that get concurrently indexed -const MAX_CONCURRENT_INDEXING: usize = 50; - -/// Index service -pub struct IndexService { - /// Writer of the account index - pub account: RwLock, - - /// Writer of the post index - pub post: RwLock, -} - -impl IndexService { - async fn add_document(&self, req: AddIndexRequest, index: &SearchIndex) -> tonic::Result<()> { - let (writer, document) = match req.index_entity { - Some(IndexEntity::Account(data)) => { - let account_schema = &index.schemas.account; - let mut document = Document::new(); - document.add_bytes(account_schema.id, data.id); - document.add_text(account_schema.username, data.username); - - if let Some(display_name) = data.display_name { - document.add_text(account_schema.display_name, display_name); - } - if let Some(description) = data.description { - document.add_text(account_schema.description, description); - } - - document.add_date( - account_schema.indexed_at, - DateTime::from_utc(OffsetDateTime::now_utc()), - ); - - increment_counter!("added_documents", "index" => GrpcSearchIndex::Account.as_str_name()); - - (self.account.read().await, document) - } - Some(IndexEntity::Post(data)) => { - let post_schema = &index.schemas.post; - let mut document = Document::new(); - document.add_bytes(post_schema.id, data.id); - document.add_text(post_schema.content, data.content); - - if let Some(subject) = data.subject { - document.add_text(post_schema.subject, subject); - } - - document.add_date( - post_schema.indexed_at, - DateTime::from_utc(OffsetDateTime::now_utc()), - ); - - increment_counter!("added_documents", "index" => GrpcSearchIndex::Post.as_str_name()); - - (self.post.read().await, document) - } - None => return Err(Status::invalid_argument("missing index data")), - }; - - writer - .add_document(document) - .map_err(|e| Status::internal(e.to_string()))?; - - Ok(()) - } - - async fn commit_all(&self) -> tonic::Result<()> { - self.account - .write() - .await - .prepare_commit() - .unwrap() - .commit_future() - .await - .map_err(|e| Status::internal(e.to_string()))?; - - self.post - .write() - .await - .prepare_commit() - .unwrap() - .commit_future() - .await - .map_err(|e| Status::internal(e.to_string()))?; - - Ok(()) - } - - async fn delete_document( - &self, - req: RemoveIndexRequest, - index: &SearchIndex, - ) -> tonic::Result<()> { - let (writer, id_field) = match req.index() { - GrpcSearchIndex::Account => { - increment_counter!("removed_documents", "index" => GrpcSearchIndex::Account.as_str_name()); - - (self.account.read().await, index.schemas.account.id) - } - GrpcSearchIndex::Post => { - increment_counter!("removed_documents", "index" => GrpcSearchIndex::Post.as_str_name()); - - (self.post.read().await, index.schemas.post.id) - } - }; - - let term = Term::from_field_bytes(id_field, &req.id); - writer.delete_term(term); - - Ok(()) - } -} - -#[async_trait] -#[autometrics(track_concurrency)] -impl Index for IndexService { - async fn add( - &self, - mut req: Request>, - ) -> tonic::Result> { - let index = req.extensions().get::().unwrap().clone(); - - req.get_mut() - .map_ok(|req| self.add_document(req, &index)) - .try_buffer_unordered(MAX_CONCURRENT_INDEXING) - .try_collect::<()>() - .await?; - - self.commit_all().await?; - - Ok(Response::new(AddIndexResponse {})) - } - - async fn remove( - &self, - mut req: Request>, - ) -> tonic::Result> { - let index = req.extensions().get::().unwrap().clone(); - - req.get_mut() - .map_ok(|req| self.delete_document(req, &index)) - .try_buffer_unordered(MAX_CONCURRENT_INDEXING) - .try_collect() - .await?; - - self.commit_all().await?; - - Ok(Response::new(RemoveIndexResponse {})) - } - - async fn reset(&self, req: Request) -> tonic::Result> { - let mut writer = match req.get_ref().index() { - GrpcSearchIndex::Account => self.account.write().await, - GrpcSearchIndex::Post => self.post.write().await, - }; - - writer - .delete_all_documents() - .map_err(|e| Status::internal(e.to_string()))?; - - writer - .prepare_commit() - .unwrap() - .commit_future() - .await - .map_err(|e| Status::internal(e.to_string()))?; - - Ok(Response::new(ResetResponse {})) - } -} diff --git a/kitsune-search-server/src/grpc/service/mod.rs b/kitsune-search-server/src/grpc/service/mod.rs deleted file mode 100644 index 640c41e8e..000000000 --- a/kitsune-search-server/src/grpc/service/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! -//! Services provided by the gRPC server -//! - -mod index; -mod search; - -pub use self::{index::IndexService, search::SearchService}; diff --git a/kitsune-search-server/src/grpc/service/search.rs b/kitsune-search-server/src/grpc/service/search.rs deleted file mode 100644 index 7a4334260..000000000 --- a/kitsune-search-server/src/grpc/service/search.rs +++ /dev/null @@ -1,89 +0,0 @@ -use crate::{ - config::Configuration, - search::{schema::PrepareQuery, SearchIndex}, -}; -use autometrics::autometrics; -use kitsune_search_proto::{ - common::SearchIndex as GrpcSearchIndex, - search::{search_server::Search, SearchRequest, SearchResponse, SearchResult}, -}; -use std::ops::Bound; -use tantivy::{collector::TopDocs, DateTime, IndexReader, Order}; -use tonic::{async_trait, Request, Response, Status}; - -/// Search service -pub struct SearchService { - /// Reader of the account index - pub account: IndexReader, - - /// Reader of the post index - pub post: IndexReader, -} - -#[async_trait] -#[autometrics(track_concurrency)] -impl Search for SearchService { - async fn search(&self, req: Request) -> tonic::Result> { - let config = req.extensions().get::().unwrap(); - let index = req.extensions().get::().unwrap(); - - let bounds = ( - req.get_ref() - .min_id - .as_deref() - .map_or(Bound::Unbounded, Bound::Included), - req.get_ref() - .max_id - .as_deref() - .map_or(Bound::Unbounded, Bound::Included), - ); - let (query, searcher, id_field) = match req.get_ref().index() { - GrpcSearchIndex::Account => ( - index.schemas.account.prepare_query( - &req.get_ref().query, - bounds, - config.levenshtein_distance, - ), - self.account.searcher(), - index.schemas.account.id, - ), - GrpcSearchIndex::Post => ( - index.schemas.post.prepare_query( - &req.get_ref().query, - bounds, - config.levenshtein_distance, - ), - self.post.searcher(), - index.schemas.post.id, - ), - }; - - let top_docs_collector = TopDocs::with_limit(req.get_ref().max_results as usize) - .and_offset(req.get_ref().offset as usize) - .order_by_fast_field::("indexed_at", Order::Desc); - let results = searcher - .search(&query, &top_docs_collector) - .map_err(|e| Status::internal(e.to_string()))?; - - let documents = results - .into_iter() - .map(|(_score, addr)| { - searcher.doc(addr).map(|doc| { - let id = doc - .get_first(id_field) - .unwrap() - .as_bytes() - .unwrap() - .to_vec(); - - SearchResult { id } - }) - }) - .collect::>() - .map_err(|err| Status::internal(err.to_string()))?; - - increment_counter!("served_search_requests"); - - Ok(Response::new(SearchResponse { results: documents })) - } -} diff --git a/kitsune-search-server/src/lib.rs b/kitsune-search-server/src/lib.rs deleted file mode 100644 index 21641b7cd..000000000 --- a/kitsune-search-server/src/lib.rs +++ /dev/null @@ -1,21 +0,0 @@ -#![doc = include_str!("../README.md")] -#![forbid(missing_docs, rust_2018_idioms, unsafe_code)] -#![warn(clippy::all, clippy::pedantic)] -#![allow( - forbidden_lint_groups, - clippy::cast_possible_truncation, - clippy::missing_errors_doc, - clippy::missing_panics_doc, - clippy::module_name_repetitions -)] - -#[macro_use] -extern crate metrics; - -#[macro_use] -extern crate tracing; - -pub mod config; -pub mod grpc; -pub mod search; -pub mod util; diff --git a/kitsune-search-server/src/main.rs b/kitsune-search-server/src/main.rs deleted file mode 100644 index 68d9ec35d..000000000 --- a/kitsune-search-server/src/main.rs +++ /dev/null @@ -1,66 +0,0 @@ -#![forbid(rust_2018_idioms, unsafe_code)] -#![warn(clippy::all, clippy::pedantic)] -#![allow( - forbidden_lint_groups, - clippy::cast_possible_truncation, - clippy::module_name_repetitions -)] - -#[macro_use] -extern crate tracing; - -use kitsune_search_server::{config::Configuration, search::SearchIndex}; -use metrics_exporter_prometheus::PrometheusBuilder; -use metrics_tracing_context::{MetricsLayer, TracingContextLayer}; -use metrics_util::layers::Layer as _; -use std::env; -use tracing::metadata::LevelFilter; -use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer as _, Registry}; - -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - -const STARTUP_FIGLET: &str = r" -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ ┃ -┃ ██╗ ██╗██╗████████╗███████╗██╗ ██╗███╗ ██╗███████╗ ███████╗███████╗ █████╗ ██████╗ ██████╗██╗ ██╗ ┃ -┃ ██║ ██╔╝██║╚══██╔══╝██╔════╝██║ ██║████╗ ██║██╔════╝ ██╔════╝██╔════╝██╔══██╗██╔══██╗██╔════╝██║ ██║ ┃ -┃ █████╔╝ ██║ ██║ ███████╗██║ ██║██╔██╗ ██║█████╗ ███████╗█████╗ ███████║██████╔╝██║ ███████║ ┃ -┃ ██╔═██╗ ██║ ██║ ╚════██║██║ ██║██║╚██╗██║██╔══╝ ╚════██║██╔══╝ ██╔══██║██╔══██╗██║ ██╔══██║ ┃ -┃ ██║ ██╗██║ ██║ ███████║╚██████╔╝██║ ╚████║███████╗ ███████║███████╗██║ ██║██║ ██║╚██████╗██║ ██║ ┃ -┃ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═══╝╚══════╝ ╚══════╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ ┃ -┃ ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -"; - -#[tokio::main] -async fn main() { - println!("{STARTUP_FIGLET}"); - - dotenvy::dotenv().ok(); - let config: Configuration = envy::from_env().unwrap(); - - let (prometheus_recorder, server_future) = PrometheusBuilder::new() - .with_http_listener(([0, 0, 0, 0], config.prometheus_port)) - .build() - .unwrap(); - tokio::spawn(server_future); - - let env_filter = env::var("RUST_LOG").map_or_else( - |_| Targets::default().with_default(LevelFilter::INFO), - |targets| targets.parse().expect("Failed to parse RUST_LOG value"), - ); - let subscriber = Registry::default() - .with(tracing_subscriber::fmt::layer().with_filter(env_filter)) - .with(MetricsLayer::new()); - tracing::subscriber::set_global_default(subscriber) - .expect("Failed to install tracing subscriber"); - - let recorder = TracingContextLayer::all().layer(prometheus_recorder); - metrics::set_boxed_recorder(Box::new(recorder)).expect("failed to install metrics recorder"); - - info!(port = config.port, "Starting up Kitsune search"); - - let index = SearchIndex::prepare(&config).unwrap(); - kitsune_search_server::grpc::start(config, index).await; -} diff --git a/kitsune-search-server/src/search/mod.rs b/kitsune-search-server/src/search/mod.rs deleted file mode 100644 index 0c8c9840c..000000000 --- a/kitsune-search-server/src/search/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! -//! Search index management -//! - -use self::schema::{AccountSchema, PostSchema}; -use crate::config::Configuration; -use std::fs; -use tantivy::{directory::MmapDirectory, Index}; - -pub mod schema; - -/// Collection of the managed search indices -#[derive(Clone)] -pub struct SearchIndices { - /// Account search index - pub account: Index, - - /// Post search index - pub post: Index, -} - -/// Collections of the schemas of the managed search indices -#[derive(Clone, Default)] -pub struct SearchSchemas { - /// Account search index schema - pub account: AccountSchema, - - /// Post search index schema - pub post: PostSchema, -} - -/// The overarching search index -/// -/// Contains all the managed schemas and indices -#[derive(Clone)] -pub struct SearchIndex { - /// Managed indices - pub indices: SearchIndices, - - /// Managed schemas - pub schemas: SearchSchemas, -} - -impl SearchIndex { - /// Create or open a search index - pub fn prepare(config: &Configuration) -> tantivy::Result { - let search_schemas = SearchSchemas::default(); - - let account_index_dir = config.index_dir_path.join("account"); - let post_index_dir = config.index_dir_path.join("post"); - - if !account_index_dir.exists() { - fs::create_dir_all(&account_index_dir)?; - } - if !post_index_dir.exists() { - fs::create_dir_all(&post_index_dir)?; - } - - let account_directory = MmapDirectory::open(account_index_dir)?; - let account_index = Index::open_or_create( - account_directory, - search_schemas.account.tantivy_schema.clone(), - )?; - - let post_directory = MmapDirectory::open(post_index_dir)?; - let post_index = - Index::open_or_create(post_directory, search_schemas.post.tantivy_schema.clone())?; - - Ok(Self { - indices: SearchIndices { - account: account_index, - post: post_index, - }, - schemas: search_schemas, - }) - } -} diff --git a/kitsune-search-server/src/search/schema.rs b/kitsune-search-server/src/search/schema.rs deleted file mode 100644 index 84ecca550..000000000 --- a/kitsune-search-server/src/search/schema.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! -//! Schemas managed by the service -//! - -use crate::util::BoundExt; -use std::ops::Bound; -use tantivy::{ - query::{BooleanQuery, FuzzyTermQuery, Query, RangeQuery}, - schema::{Field, Schema, Type, FAST, INDEXED, STORED, STRING, TEXT}, - Term, -}; - -/// Bounds alias used in the [`PrepareQuery`] trait -/// -/// Contains the lower and upper bound of a byte slice range -pub type Bounds<'a> = (Bound<&'a [u8]>, Bound<&'a [u8]>); - -/// Prepare a range query that matches the provided byte field with some lower and upper bounds -fn prepare_range_query(schema: &Schema, field: Field, (left, right): Bounds<'_>) -> RangeQuery { - // We vendored the exactly same API, there will be no change in behaviour - #[allow(unstable_name_collisions)] - let (left, right) = ( - left.map(|val| Term::from_field_bytes(field, val)), - right.map(|val| Term::from_field_bytes(field, val)), - ); - - RangeQuery::new_term_bounds( - schema.get_field_name(field).to_string(), - Type::Bytes, - &left, - &right, - ) -} - -/// Trait for preparing a tantivy query for some schema -pub trait PrepareQuery { - /// Type of the returned query - type Query: Query; - - /// Prepare a tantivy query - fn prepare_query( - &self, - query: &str, - id_bounds: Bounds<'_>, - levenshtein_distance: u8, - ) -> Self::Query; -} - -/// Account search schema -#[derive(Clone)] -pub struct AccountSchema { - /// ID field (contains a UUID) - pub id: Field, - - /// Display name field (might be empty) - pub display_name: Field, - - /// Username field - pub username: Field, - - /// Description (or bio, etc.) field (might be empty) - pub description: Field, - - /// Unix timestamp of when the account was indexed - pub indexed_at: Field, - - /// The underlying tantivy schema with the above defined fields - pub tantivy_schema: Schema, -} - -impl Default for AccountSchema { - fn default() -> Self { - let mut builder = Schema::builder(); - let id = builder.add_bytes_field("id", FAST | INDEXED | STORED); - let display_name = builder.add_text_field("display_name", FAST | TEXT); - let username = builder.add_text_field("username", FAST | STRING); - let description = builder.add_text_field("description", FAST | TEXT); - let indexed_at = builder.add_date_field("indexed_at", FAST); - let tantivy_schema = builder.build(); - - Self { - id, - display_name, - username, - description, - indexed_at, - tantivy_schema, - } - } -} - -impl PrepareQuery for AccountSchema { - type Query = BooleanQuery; - - fn prepare_query( - &self, - query: &str, - id_bounds: Bounds<'_>, - levenshtein_distance: u8, - ) -> Self::Query { - let queries: Vec> = vec![ - Box::new(FuzzyTermQuery::new( - Term::from_field_text(self.display_name, query), - levenshtein_distance, - true, - )), - Box::new(FuzzyTermQuery::new( - Term::from_field_text(self.username, query), - levenshtein_distance, - true, - )), - Box::new(FuzzyTermQuery::new( - Term::from_field_text(self.description, query), - levenshtein_distance, - true, - )), - ]; - - BooleanQuery::intersection(vec![ - Box::new(BooleanQuery::union(queries)), - Box::new(prepare_range_query( - &self.tantivy_schema, - self.id, - id_bounds, - )), - ]) - } -} - -/// Post search schema -#[derive(Clone)] -pub struct PostSchema { - /// ID field (contains a UUID) - pub id: Field, - - /// Subject field (might be empty) - pub subject: Field, - - /// Content field - pub content: Field, - - /// Unix timestamp of when the post was indexed - pub indexed_at: Field, - - /// The underlying tantivy schema with the above defined fields - pub tantivy_schema: Schema, -} - -impl Default for PostSchema { - fn default() -> Self { - let mut builder = Schema::builder(); - let id = builder.add_bytes_field("id", FAST | INDEXED | STORED); - let subject = builder.add_text_field("subject", FAST | TEXT); - let content = builder.add_text_field("content", FAST | TEXT); - let indexed_at = builder.add_date_field("indexed_at", FAST); - let tantivy_schema = builder.build(); - - Self { - id, - subject, - content, - indexed_at, - tantivy_schema, - } - } -} - -impl PrepareQuery for PostSchema { - type Query = BooleanQuery; - - fn prepare_query( - &self, - query: &str, - id_bounds: Bounds<'_>, - levenshtein_distance: u8, - ) -> Self::Query { - let queries: Vec> = vec![ - Box::new(FuzzyTermQuery::new( - Term::from_field_text(self.subject, query), - levenshtein_distance, - true, - )), - Box::new(FuzzyTermQuery::new( - Term::from_field_text(self.content, query), - levenshtein_distance, - true, - )), - ]; - - BooleanQuery::intersection(vec![ - Box::new(BooleanQuery::union(queries)), - Box::new(prepare_range_query( - &self.tantivy_schema, - self.id, - id_bounds, - )), - ]) - } -} diff --git a/kitsune-search-server/src/util.rs b/kitsune-search-server/src/util.rs deleted file mode 100644 index 78e12855a..000000000 --- a/kitsune-search-server/src/util.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! -//! Utility functions -//! - -use std::ops::Bound; - -/// Extension trait on [`Bound`] -pub trait BoundExt { - /// Vendored until [rust-lang/rust#86026](https://github.com/rust-lang/rust/issues/86026) is stabilised - fn map U>(self, f: F) -> Bound; -} - -impl BoundExt for Bound { - #[inline] - fn map U>(self, f: F) -> Bound { - match self { - Bound::Unbounded => Bound::Unbounded, - Bound::Included(x) => Bound::Included(f(x)), - Bound::Excluded(x) => Bound::Excluded(f(x)), - } - } -} - -/// Vendored until [rust-lang/rust#88581](https://github.com/rust-lang/rust/issues/88581) is stabilised -#[inline] -#[must_use] -pub const fn div_ceil(lhs: usize, rhs: usize) -> usize { - let d = lhs / rhs; - let r = lhs % rhs; - if r > 0 && rhs > 0 { - d + 1 - } else { - d - } -} diff --git a/kitsune-search-server/tests/account.rs b/kitsune-search-server/tests/account.rs deleted file mode 100644 index bb4e77f48..000000000 --- a/kitsune-search-server/tests/account.rs +++ /dev/null @@ -1,77 +0,0 @@ -use self::common::TestClient; -use futures_util::stream; -use kitsune_search_proto::{ - common::SearchIndex, - index::{add_index_request::IndexEntity, AddAccountIndex, AddIndexRequest, RemoveIndexRequest}, - search::SearchRequest, -}; -use std::{future, time::Duration}; - -mod common; - -#[tokio::test] -async fn index_search_remove() { - let mut test_client = TestClient::create().await; - - let id: [u8; 24] = rand::random(); - test_client - .index - .add(stream::once(future::ready(AddIndexRequest { - index_entity: Some(IndexEntity::Account(AddAccountIndex { - id: id.to_vec(), - display_name: Some("name".into()), - username: "cool_username".into(), - description: Some("Really cool test account. Very important".into()), - })), - }))) - .await - .unwrap(); - - tokio::time::sleep(Duration::from_secs(1)).await; // Wait until the write as propagated to the reader - - let response = test_client - .search - .search(SearchRequest { - index: SearchIndex::Account.into(), - query: "tset".into(), - max_results: 20, - min_id: None, - max_id: None, - offset: 0, - }) - .await - .unwrap() - .into_inner(); - - assert_eq!(response.results.len(), 1); - assert_eq!(response.results[0].id, id); - - // -- Remove the account from the index -- - - test_client - .index - .remove(stream::once(future::ready(RemoveIndexRequest { - index: SearchIndex::Account.into(), - id: id.to_vec(), - }))) - .await - .unwrap(); - - tokio::time::sleep(Duration::from_secs(1)).await; // Wait until the write as propagated to the reader - - let response = test_client - .search - .search(SearchRequest { - index: SearchIndex::Account.into(), - query: "tset".into(), - max_results: 20, - min_id: None, - max_id: None, - offset: 0, - }) - .await - .unwrap() - .into_inner(); - - assert!(response.results.is_empty(), "{response:#?}"); -} diff --git a/kitsune-search-server/tests/common.rs b/kitsune-search-server/tests/common.rs deleted file mode 100644 index 38491d6da..000000000 --- a/kitsune-search-server/tests/common.rs +++ /dev/null @@ -1,40 +0,0 @@ -use kitsune_search_proto::{index::index_client::IndexClient, search::search_client::SearchClient}; -use kitsune_search_server::{config::Configuration, search::SearchIndex}; -use rand::Rng; -use tempfile::TempDir; -use tonic::transport::Channel; - -pub struct TestClient { - pub index: IndexClient, - pub search: SearchClient, - _temp_dir: TempDir, -} - -impl TestClient { - pub async fn create() -> Self { - let port = rand::thread_rng().gen_range(1025..u16::MAX); - let temp_dir = TempDir::new().unwrap(); - - let config = Configuration { - index_dir_path: temp_dir.path().into(), - levenshtein_distance: 2, - memory_arena_size: "3MB".parse().unwrap(), - port, - prometheus_port: 0, // Doesn't matter. No Prometheus server gets started - read_only: false, - }; - - let search_index = SearchIndex::prepare(&config).unwrap(); - tokio::spawn(kitsune_search_server::grpc::start(config, search_index)); - - Self { - index: IndexClient::connect(format!("http://localhost:{port}")) - .await - .unwrap(), - search: SearchClient::connect(format!("http://localhost:{port}")) - .await - .unwrap(), - _temp_dir: temp_dir, - } - } -} diff --git a/kitsune-search-server/tests/post.rs b/kitsune-search-server/tests/post.rs deleted file mode 100644 index 3639aba37..000000000 --- a/kitsune-search-server/tests/post.rs +++ /dev/null @@ -1,106 +0,0 @@ -use self::common::TestClient; -use futures_util::stream; -use kitsune_search_proto::{ - common::SearchIndex, - index::{add_index_request::IndexEntity, AddIndexRequest, AddPostIndex, RemoveIndexRequest}, - search::SearchRequest, -}; -use std::{future, time::Duration}; - -mod common; - -const POST_CONTENT: &str = r#" -Lorem ipsum dolor sit amet, consectetur adipiscing elit. -Fusce hendrerit consequat tellus sed rhoncus. Nullam nec ultrices tellus. -In vitae tempus mauris, vel auctor orci. -In porttitor, lectus sed consectetur blandit, ligula neque efficitur eros, sit amet ullamcorper nunc odio eu sem. -Quisque luctus rutrum ullamcorper. -Nam vitae varius turpis, eget porta enim. -Vestibulum luctus ex id ipsum mattis porttitor. -Duis faucibus risus quis varius cursus. -Praesent et mi orci. -Curabitur id fermentum elit, sed congue lacus. -Pellentesque id augue vitae tortor vehicula placerat vestibulum et justo. - -Praesent quis ex magna. Ut congue dapibus tortor quis dignissim. -Vestibulum sit amet nulla faucibus, faucibus felis in, ultrices odio. -Cras eu tellus at eros molestie condimentum. -Phasellus ornare est a ante blandit, eget commodo odio dapibus. -Praesent enim mauris, consectetur quis rutrum quis, congue sit amet ligula. -Vestibulum finibus tincidunt ipsum, id luctus lorem tincidunt nec. -Duis nec arcu at libero finibus finibus eget a elit. -Nullam a massa ornare nisl tristique blandit vitae vitae nunc. -Suspendisse faucibus pellentesque risus, id tincidunt urna vehicula a. -Maecenas at justo auctor metus varius ultrices. -Nullam feugiat dictum tortor, ac vestibulum dolor bibendum nec. -Phasellus varius vehicula lectus. -Duis vestibulum nisi turpis, et pharetra massa sollicitudin tempor. -Vivamus risus dolor, venenatis a convallis eget, tincidunt sed lacus. -Sed tincidunt eros gravida, sollicitudin enim ut, pellentesque libero. -"#; - -#[tokio::test] -async fn index_search_remove() { - let mut test_client = TestClient::create().await; - - let id: [u8; 24] = rand::random(); - test_client - .index - .add(stream::once(future::ready(AddIndexRequest { - index_entity: Some(IndexEntity::Post(AddPostIndex { - id: id.to_vec(), - subject: None, - content: POST_CONTENT.to_string(), - })), - }))) - .await - .unwrap(); - - tokio::time::sleep(Duration::from_secs(1)).await; - - let response = test_client - .search - .search(SearchRequest { - index: SearchIndex::Post.into(), - query: "lroem".into(), - max_results: 20, - min_id: None, - max_id: None, - offset: 0, - }) - .await - .unwrap() - .into_inner(); - - assert_eq!(response.results.len(), 1); - assert_eq!(response.results[0].id, id); - - // -- Remove post from the index -- - - test_client - .index - .remove(stream::once(future::ready(RemoveIndexRequest { - index: SearchIndex::Post.into(), - id: id.to_vec(), - }))) - .await - .unwrap(); - - tokio::time::sleep(Duration::from_secs(1)).await; - - let response = test_client - .search - .search(SearchRequest { - index: SearchIndex::Post.into(), - query: "lroem".into(), - max_results: 20, - min_id: None, - max_id: None, - offset: 0, - }) - .await - .unwrap() - .into_inner(); - - assert!(response.results.is_empty()); -} diff --git a/kitsune/Cargo.toml b/kitsune/Cargo.toml index 86893bb0b..d46772cc8 100644 --- a/kitsune/Cargo.toml +++ b/kitsune/Cargo.toml @@ -126,7 +126,6 @@ graphql-api = [ "dep:async-graphql-axum", "speedy-uuid/async-graphql", ] -kitsune-search = ["kitsune-core/kitsune-search"] mastodon-api = ["kitsune-core/mastodon-api"] meilisearch = ["kitsune-core/meilisearch"] metrics = [ diff --git a/proto/search/common.proto b/proto/search/common.proto deleted file mode 100644 index 2192cc29b..000000000 --- a/proto/search/common.proto +++ /dev/null @@ -1,16 +0,0 @@ -syntax = "proto3"; - -package kitsune.common; - -// --- Start: Type definitions --- - -// Search index -enum SearchIndex { - // Account search index - Account = 0; - - // Post search index - Post = 1; -} - -// --- End: Type definitions --- diff --git a/proto/search/index.proto b/proto/search/index.proto deleted file mode 100644 index a11e7bfdc..000000000 --- a/proto/search/index.proto +++ /dev/null @@ -1,86 +0,0 @@ -syntax = "proto3"; - -package kitsune.index; - -import "common.proto"; - -// --- Start: Message definitions --- - -// Request to add an account to the search index -message AddAccountIndex { - // Unique identifier of the account (UUID) - bytes id = 1; - - // Display name - optional string display_name = 2; - - // Username - string username = 3; - - // Description - optional string description = 4; -} - -// Request to add a post to the search index -message AddPostIndex { - // Unique identifier of the post (UUID) - bytes id = 1; - - // Subject - optional string subject = 2; - - // Content - string content = 3; -} - -// Request to add a new entry to the search index -message AddIndexRequest { - // Entity that is added to the search index - oneof index_entity { - // Account that's being added - AddAccountIndex account = 1; - - // Post that's being added - AddPostIndex post = 2; - } -} - -// Response to an add request -message AddIndexResponse {} - -// Request to remove an entry from the search index by its ID -message RemoveIndexRequest { - // Index from which an entry should get removed - kitsune.common.SearchIndex index = 1; - - // Unique identifier of the entry (UUID) - bytes id = 2; -} - -// Response to a remove request -message RemoveIndexResponse {} - -// Request to completely reset a search index (aka. delete all its entries) -message ResetRequest { - // Search index that should be reset - kitsune.common.SearchIndex index = 1; -} - -// Response to a reset request -message ResetResponse {} - -// --- End: Message definitions --- - -// Indexing service -service Index { - // Add a document to the search index - rpc Add(stream AddIndexRequest) returns (AddIndexResponse); - - // Remove a document from the search index - rpc Remove(stream RemoveIndexRequest) returns (RemoveIndexResponse); - - // Reset the entire index - // - // Warning: This is a major destructive operation. It will delete all entries of the index. - rpc Reset(ResetRequest) returns (ResetResponse); -} diff --git a/proto/search/search.proto b/proto/search/search.proto deleted file mode 100644 index 23103063e..000000000 --- a/proto/search/search.proto +++ /dev/null @@ -1,48 +0,0 @@ -syntax = "proto3"; - -package kitsune.search; - -import "common.proto"; - -// --- Start: Message definitions --- - -// One result of a search -message SearchResult { - // Unique identifier of the entry - bytes id = 1; -} - -// Request to query the search index -message SearchRequest { - // Search index to search through - kitsune.common.SearchIndex index = 1; - - // Query for searching through the entries - string query = 2; - - // Maximum number of results - uint64 max_results = 3; - - // Minimum ID - optional bytes min_id = 4; - - // Maximum ID - optional bytes max_id = 5; - - // Offset - uint64 offset = 6; -} - -// Response to a search request -message SearchResponse { - // List of search results - repeated SearchResult results = 1; -} - -// --- End: Message definitions --- - -// Search service -service Search { - // Search through all the entries - rpc Search(SearchRequest) returns (SearchResponse); -}