From f578155602343136089ac034a4d154399ce58b6b Mon Sep 17 00:00:00 2001 From: liyang Date: Fri, 16 Aug 2024 11:11:20 +0800 Subject: [PATCH] feat: add GcsConfig credential field (#4568) --- .env.example | 7 ++++--- config/config.md | 2 ++ config/datanode.example.toml | 6 ++++++ config/standalone.example.toml | 6 ++++++ src/datanode/src/config.rs | 4 ++++ src/datanode/src/store/gcs.rs | 1 + src/object-store/tests/object_store_test.rs | 1 + tests-integration/src/test_util.rs | 2 ++ 8 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index d463d85a7d16..6410218664a9 100644 --- a/.env.example +++ b/.env.example @@ -14,10 +14,11 @@ GT_AZBLOB_CONTAINER=AZBLOB container GT_AZBLOB_ACCOUNT_NAME=AZBLOB account name GT_AZBLOB_ACCOUNT_KEY=AZBLOB account key GT_AZBLOB_ENDPOINT=AZBLOB endpoint -# Settings for gcs test -GT_GCS_BUCKET = GCS bucket +# Settings for gcs test +GT_GCS_BUCKET = GCS bucket GT_GCS_SCOPE = GCS scope -GT_GCS_CREDENTIAL_PATH = GCS credential path +GT_GCS_CREDENTIAL_PATH = GCS credential path +GT_GCS_CREDENTIAL = GCS credential GT_GCS_ENDPOINT = GCS end point # Settings for kafka wal test GT_KAFKA_ENDPOINTS = localhost:9092 diff --git a/config/config.md b/config/config.md index dfd2ab889c6d..cf622daf9f59 100644 --- a/config/config.md +++ b/config/config.md @@ -98,6 +98,7 @@ | `storage.account_key` | String | `None` | The account key of the azure account.
**It's only used when the storage type is `Azblob`**. | | `storage.scope` | String | `None` | The scope of the google cloud storage.
**It's only used when the storage type is `Gcs`**. | | `storage.credential_path` | String | `None` | The credential path of the google cloud storage.
**It's only used when the storage type is `Gcs`**. | +| `storage.credential` | String | `None` | The credential of the google cloud storage.
**It's only used when the storage type is `Gcs`**. | | `storage.container` | String | `None` | The container of the azure account.
**It's only used when the storage type is `Azblob`**. | | `storage.sas_token` | String | `None` | The sas token of the azure account.
**It's only used when the storage type is `Azblob`**. | | `storage.endpoint` | String | `None` | The endpoint of the S3 service.
**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. | @@ -387,6 +388,7 @@ | `storage.account_key` | String | `None` | The account key of the azure account.
**It's only used when the storage type is `Azblob`**. | | `storage.scope` | String | `None` | The scope of the google cloud storage.
**It's only used when the storage type is `Gcs`**. | | `storage.credential_path` | String | `None` | The credential path of the google cloud storage.
**It's only used when the storage type is `Gcs`**. | +| `storage.credential` | String | `None` | The credential of the google cloud storage.
**It's only used when the storage type is `Gcs`**. | | `storage.container` | String | `None` | The container of the azure account.
**It's only used when the storage type is `Azblob`**. | | `storage.sas_token` | String | `None` | The sas token of the azure account.
**It's only used when the storage type is `Azblob`**. | | `storage.endpoint` | String | `None` | The endpoint of the S3 service.
**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 81cbc4703c4d..c5cc04ebcadc 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -241,6 +241,7 @@ backoff_deadline = "5mins" # root = "data" # scope = "test" # credential_path = "123456" +# credential = "base64-credential" # endpoint = "https://storage.googleapis.com" ## The data storage options. @@ -312,6 +313,11 @@ scope = "test" ## +toml2docs:none-default credential_path = "test" +## The credential of the google cloud storage. +## **It's only used when the storage type is `Gcs`**. +## +toml2docs:none-default +credential= "base64-credential" + ## The container of the azure account. ## **It's only used when the storage type is `Azblob`**. ## +toml2docs:none-default diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 43f36373706b..ffc2de5e647c 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -286,6 +286,7 @@ retry_delay = "500ms" # root = "data" # scope = "test" # credential_path = "123456" +# credential = "base64-credential" # endpoint = "https://storage.googleapis.com" ## The data storage options. @@ -357,6 +358,11 @@ scope = "test" ## +toml2docs:none-default credential_path = "test" +## The credential of the google cloud storage. +## **It's only used when the storage type is `Gcs`**. +## +toml2docs:none-default +credential = "base64-credential" + ## The container of the azure account. ## **It's only used when the storage type is `Azblob`**. ## +toml2docs:none-default diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs index 8d99c6faf25b..6ce16b779922 100644 --- a/src/datanode/src/config.rs +++ b/src/datanode/src/config.rs @@ -179,6 +179,8 @@ pub struct GcsConfig { pub scope: String, #[serde(skip_serializing)] pub credential_path: SecretString, + #[serde(skip_serializing)] + pub credential: SecretString, pub endpoint: String, #[serde(flatten)] pub cache: ObjectStorageCacheConfig, @@ -190,6 +192,7 @@ impl PartialEq for GcsConfig { && self.bucket == other.bucket && self.scope == other.scope && self.credential_path.expose_secret() == other.credential_path.expose_secret() + && self.credential.expose_secret() == other.credential.expose_secret() && self.endpoint == other.endpoint && self.cache == other.cache } @@ -243,6 +246,7 @@ impl Default for GcsConfig { bucket: String::default(), scope: String::default(), credential_path: SecretString::from(String::default()), + credential: SecretString::from(String::default()), endpoint: String::default(), cache: ObjectStorageCacheConfig::default(), } diff --git a/src/datanode/src/store/gcs.rs b/src/datanode/src/store/gcs.rs index 6c68ad3baccc..f982ec5f1578 100644 --- a/src/datanode/src/store/gcs.rs +++ b/src/datanode/src/store/gcs.rs @@ -34,6 +34,7 @@ pub(crate) async fn new_gcs_object_store(gcs_config: &GcsConfig) -> Result Result<()> { .bucket(&env::var("GT_GCS_BUCKET").unwrap()) .scope(&env::var("GT_GCS_SCOPE").unwrap()) .credential_path(&env::var("GT_GCS_CREDENTIAL_PATH").unwrap()) + .credential(&env::var("GT_GCS_CREDENTIAL").unwrap()) .endpoint(&env::var("GT_GCS_ENDPOINT").unwrap()); let store = ObjectStore::new(builder).unwrap().finish(); diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index cefd8bed6b94..07237c8bc1b3 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -160,6 +160,7 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te bucket: env::var("GT_GCS_BUCKET").unwrap(), scope: env::var("GT_GCS_SCOPE").unwrap(), credential_path: env::var("GT_GCS_CREDENTIAL_PATH").unwrap().into(), + credential: env::var("GT_GCS_CREDENTIAL").unwrap().into(), endpoint: env::var("GT_GCS_ENDPOINT").unwrap(), ..Default::default() }; @@ -169,6 +170,7 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te .bucket(&gcs_config.bucket) .scope(&gcs_config.scope) .credential_path(gcs_config.credential_path.expose_secret()) + .credential(gcs_config.credential.expose_secret()) .endpoint(&gcs_config.endpoint); let config = ObjectStoreConfig::Gcs(gcs_config);