diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0034ac81956..00e495bf99f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,7 +37,7 @@ jobs: - "4571:4571" - "8080:8080" env: - SERVICES: kinesis,s3 + SERVICES: kinesis,s3,sqs options: >- --health-cmd "curl -k https://localhost:4566" --health-interval 10s diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index e15ada45215..75aad196108 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -1153,9 +1153,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" dependencies = [ "serde", ] @@ -2048,12 +2048,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dotenv" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" - [[package]] name = "dotenvy" version = "0.15.7" @@ -6217,7 +6211,7 @@ dependencies = [ "async-trait", "bytes", "bytesize", - "dotenv", + "dotenvy", "futures", "http 0.2.12", "itertools 0.13.0", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 48a059f2173..d1c9a4f5f49 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -101,7 +101,7 @@ console-subscriber = "0.1.8" criterion = { version = "0.5", features = ["async_tokio"] } cron = "0.12.0" dialoguer = "0.10.3" -dotenv = "0.15" +dotenvy = "0.15" dyn-clone = "1.0.10" enum-iterator = "1.5" env_logger = "0.10" diff --git a/quickwit/deny.toml b/quickwit/deny.toml index 0c6c498786b..139785b1c9a 100644 --- a/quickwit/deny.toml +++ b/quickwit/deny.toml @@ -9,6 +9,7 @@ # The values provided in this template are the default values that will be used # when any section or field is not specified in your own configuration +[graph] # If 1 or more target triples (and optionally, target_features) are specified, # only the specified targets will be checked when running `cargo deny check`. # This means, if a particular package is only ever used as a target specific @@ -31,42 +32,22 @@ targets = [ # More documentation for the advisories section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html [advisories] +version = 2 # The path where the advisory database is cloned/fetched into db-path = "~/.cargo/advisory-db" # The url(s) of the advisory databases to use db-urls = ["https://github.com/rustsec/advisory-db"] -# The lint level for security vulnerabilities -vulnerability = "deny" -# The lint level for unmaintained crates -unmaintained = "warn" -# The lint level for crates that have been yanked from their source registry -yanked = "warn" -# The lint level for crates with security notices. Note that as of -# 2019-12-17 there are no security notice advisories in -# https://github.com/rustsec/advisory-db -notice = "warn" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - # TODO Remove me after rsa gets patched and released. - "RUSTSEC-2023-0071" + "RUSTSEC-2021-0153", # `encoding` is unmaintained, it's used in lindera ] -# Threshold for security vulnerabilities, any vulnerability with a CVSS score -# lower than the range specified will be ignored. Note that ignored advisories -# will still output a note when they are encountered. -# * None - CVSS Score 0.0 -# * Low - CVSS Score 0.1 - 3.9 -# * Medium - CVSS Score 4.0 - 6.9 -# * High - CVSS Score 7.0 - 8.9 -# * Critical - CVSS Score 9.0 - 10.0 -#severity-threshold = # This section is considered when running `cargo deny check licenses` # More documentation for the licenses section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -# The lint level for crates which do not have a detectable license -unlicensed = "deny" +version = 2 # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -86,26 +67,6 @@ allow = [ "Zlib", "zlib-acknowledgement", ] -# List of explicitly disallowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. -deny = [ - #"Nokia", -] -# Lint level for licenses considered copyleft -copyleft = "warn" -# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses -# * both - The license will be approved if it is both OSI-approved *AND* FSF -# * either - The license will be approved if it is either OSI-approved *OR* FSF -# * osi-only - The license will be approved if is OSI-approved *AND NOT* FSF -# * fsf-only - The license will be approved if is FSF *AND NOT* OSI-approved -# * neither - This predicate is ignored and the default lint level is used -allow-osi-fsf-free = "neither" -# Lint level used when no other predicates are matched -# 1. License isn't in the allow or deny lists -# 2. License isn't copyleft -# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" -default = "deny" # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the # canonical license text of a valid SPDX license file. @@ -114,20 +75,30 @@ confidence-threshold = 0.8 # Allow 1 or more licenses on a per-crate basis, so that particular licenses # aren't accepted for every possible crate as with the normal allow list exceptions = [ + { name = "quickwit-actors", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-aws", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-cli", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-cluster", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-codegen", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-codegen-example", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-common", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-config", allow = ["AGPL-3.0"], version = "*" }, - { name = "quickwit-index-management", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-control-plane", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-datetime", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-directories", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-doc-mapper", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-indexing", allow = ["AGPL-3.0"], version = "*" }, - { name = "quickwit-ingest-api", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-index-management", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-ingest", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-integration-tests", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-jaeger", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-janitor", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-lambda", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-macros", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-metastore", allow = ["AGPL-3.0"], version = "*" }, - { name = "quickwit-metastore-utils", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-opentelemetry", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-proto", allow = ["AGPL-3.0"], version = "*" }, + { name = "quickwit-query", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-rest-client", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-search", allow = ["AGPL-3.0"], version = "*" }, { name = "quickwit-serve", allow = ["AGPL-3.0"], version = "*" }, @@ -224,8 +195,8 @@ allow-git = [] [sources.allow-org] # 1 or more github.com organizations to allow git sources for -github = [""] +github = ["quickwit-oss"] # 1 or more gitlab.com organizations to allow git sources for -gitlab = [""] +gitlab = [] # 1 or more bitbucket.org organizations to allow git sources for -bitbucket = [""] +bitbucket = [] diff --git a/quickwit/quickwit-metastore/Cargo.toml b/quickwit/quickwit-metastore/Cargo.toml index 1f2b3d87ce8..612ada72d52 100644 --- a/quickwit/quickwit-metastore/Cargo.toml +++ b/quickwit/quickwit-metastore/Cargo.toml @@ -47,7 +47,7 @@ quickwit-query = { workspace = true } quickwit-storage = { workspace = true } [dev-dependencies] -dotenv = { workspace = true } +dotenvy = { workspace = true } futures = { workspace = true } md5 = { workspace = true } mockall = { workspace = true } diff --git a/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs b/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs index 133f1413b1e..786e4f1a4af 100644 --- a/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs +++ b/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs @@ -1744,7 +1744,7 @@ impl crate::tests::DefaultForTest for PostgresqlMetastore { // The number of connections to Postgres should not be // too catastrophic, as it is limited by the number of concurrent // unit tests running (= number of test-threads). - dotenv::dotenv().ok(); + dotenvy::dotenv().ok(); let uri: Uri = std::env::var("QW_TEST_DATABASE_URL") .expect("environment variable `QW_TEST_DATABASE_URL` should be set") .parse() diff --git a/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml b/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml index 1df59efd2c8..f81c2215f40 100644 --- a/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml +++ b/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml @@ -375,4 +375,21 @@ expected: aggregations: response_stats: sum_of_squares: 55300.0 - +# Test term aggs number precision +method: [GET] +engines: + - quickwit +endpoint: _elastic/aggregations/_search +json: + query: { match_all: {} } + size: 0 + aggs: + names: + terms: + field: "high_prec_test" +expected: + aggregations: + names: + buckets: + - doc_count: 1 + key: 1769070189829214200 diff --git a/quickwit/rest-api-tests/scenarii/aggregations/0002-doc-len.yaml b/quickwit/rest-api-tests/scenarii/aggregations/0002-doc-len.yaml index e98e59a156c..08b437015b9 100644 --- a/quickwit/rest-api-tests/scenarii/aggregations/0002-doc-len.yaml +++ b/quickwit/rest-api-tests/scenarii/aggregations/0002-doc-len.yaml @@ -12,7 +12,7 @@ json: expected: aggregations: doc_len: - value: 913.0 + value: 952.0 --- # Test doc len isn't shown when querying documents method: [GET] diff --git a/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml index 11ee82ec67f..60580ca576a 100644 --- a/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml @@ -25,6 +25,9 @@ json: - rfc3339 fast_precision: seconds fast: true + - name: high_prec_test + type: u64 + fast: true store_document_size: true --- # Create empty index @@ -64,7 +67,7 @@ endpoint: aggregations/ingest params: commit: force ndjson: - - {"name": "Fritz", "response": 30, "id": 0} + - {"name": "Fritz", "high_prec_test": 1769070189829214200, "response": 30, "id": 0} - {"name": "Fritz", "response": 30, "id": 0} - {"name": "Holger", "response": 30, "id": 4, "date": "2015-02-06T00:00:00Z", "host": "192.168.0.10"} - {"name": "Werner", "response": 20, "id": 5, "date": "2015-01-02T00:00:00Z", "host": "192.168.0.10"}