From 2944a62afee67a64d09225773583d3d08d1548f4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Fri, 2 Jun 2023 17:07:27 +0200 Subject: [PATCH 1/8] CDF-18848: [draft] try new creds in tests - [] need to prepare CDF group access for it to work properly [CDF-18848] --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 627f0dd75..372782fcc 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -40,9 +40,9 @@ jobs: TEST_OIDC_READ_CLIENT_ID: ${{ secrets.TEST_OIDC_READ_CLIENT_ID }} TEST_OIDC_READ_CLIENT_SECRET: ${{ secrets.TEST_OIDC_READ_CLIENT_SECRET }} TEST_OIDC_READ_TENANT: ${{ secrets.TEST_OIDC_READ_TENANT }} - TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID }} - TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET }} - TEST_AAD_TENANT_BLUEFIELD: "b86328db-09aa-4f0e-9a03-0136f604d20a" + TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} + TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} + TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_BLUEFIELD_2 }} AIZE_CLIENT_ID: ${{ secrets.AIZE_CLIENT_ID }} AIZE_CLIENT_SECRET: ${{ secrets.AIZE_CLIENT_SECRET }} run: | From 7fbb7696421d8ec896ec06d4238718eedb68f7e9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Fri, 2 Jun 2023 17:36:20 +0200 Subject: [PATCH 2/8] Update .github/workflows/main.yaml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 372782fcc..d619a1eb8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -42,7 +42,7 @@ jobs: TEST_OIDC_READ_TENANT: ${{ secrets.TEST_OIDC_READ_TENANT }} TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} - TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_BLUEFIELD_2 }} + TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} AIZE_CLIENT_ID: ${{ secrets.AIZE_CLIENT_ID }} AIZE_CLIENT_SECRET: ${{ secrets.AIZE_CLIENT_SECRET }} run: | From 1770f67f7b2ab9a306c4683377371e022f8e6ea8 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Fri, 2 Jun 2023 23:36:43 +0200 Subject: [PATCH 3/8] Update main.yaml --- .github/workflows/main.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d619a1eb8..16a334be5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -36,15 +36,12 @@ jobs: ./secrets/decrypt.sh secrets/secring.asc.gpg ~/.sbt/gpg/secring.asc - name: Run tests env: - TEST_API_KEY_WRITE: ${{ secrets.TEST_API_KEY_WRITE }} - TEST_OIDC_READ_CLIENT_ID: ${{ secrets.TEST_OIDC_READ_CLIENT_ID }} - TEST_OIDC_READ_CLIENT_SECRET: ${{ secrets.TEST_OIDC_READ_CLIENT_SECRET }} - TEST_OIDC_READ_TENANT: ${{ secrets.TEST_OIDC_READ_TENANT }} + TEST_OIDC_READ_CLIENT_ID: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} + TEST_OIDC_READ_CLIENT_SECRET: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} + TEST_OIDC_READ_TENANT: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} - AIZE_CLIENT_ID: ${{ secrets.AIZE_CLIENT_ID }} - AIZE_CLIENT_SECRET: ${{ secrets.AIZE_CLIENT_SECRET }} run: | TEST="test +Test/compile" if [ ${{github.ref }} == "refs/heads/master" ]; then From cebee4623f363907edfd52165d71bbaf39f0a359 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Sat, 3 Jun 2023 00:09:05 +0200 Subject: [PATCH 4/8] more --- .github/workflows/main.yaml | 5 +- .../spark/v1/DataPointsRelationTest.scala | 1 - .../scala/cognite/spark/v1/SdkV1RddTest.scala | 9 +++- .../scala/cognite/spark/v1/SparkTest.scala | 46 ++++--------------- 4 files changed, 20 insertions(+), 41 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 16a334be5..cc11ef89f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -36,12 +36,11 @@ jobs: ./secrets/decrypt.sh secrets/secring.asc.gpg ~/.sbt/gpg/secring.asc - name: Run tests env: - TEST_OIDC_READ_CLIENT_ID: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} - TEST_OIDC_READ_CLIENT_SECRET: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} - TEST_OIDC_READ_TENANT: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} + TEST_PROJECT: "spark-datasource-bluefield-tests" + TEST_CLUSTER: "bluefield.cognitedata.com" run: | TEST="test +Test/compile" if [ ${{github.ref }} == "refs/heads/master" ]; then diff --git a/src/test/scala/cognite/spark/v1/DataPointsRelationTest.scala b/src/test/scala/cognite/spark/v1/DataPointsRelationTest.scala index 0c8b8d393..8000d0614 100644 --- a/src/test/scala/cognite/spark/v1/DataPointsRelationTest.scala +++ b/src/test/scala/cognite/spark/v1/DataPointsRelationTest.scala @@ -1061,7 +1061,6 @@ class DataPointsRelationTest val df = spark.read .format("cognite.spark.v1") - .option("apiKey", jetfiretest2ApiKey) .option("type", "datapoints") .option("collectMetrics", "true") .option("metricsPrefix", metricsPrefix) diff --git a/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala b/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala index 8b9b795dc..31790f23e 100644 --- a/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala +++ b/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala @@ -1,7 +1,7 @@ package cognite.spark.v1 import cats.effect.IO -import com.cognite.sdk.scala.common.CdpApiException +import com.cognite.sdk.scala.common.{CdpApiException, OAuth2} import com.cognite.sdk.scala.v1.{Event, GenericClient} import fs2.{Chunk, Stream} import org.apache.spark.TaskContext @@ -33,6 +33,13 @@ class SdkV1RddTest extends FlatSpec with Matchers with ParallelTestExecution wit implicit val implicitBackend: SttpBackend[IO, Any] = CdpConnector.retryingSttpBackend(3, 5) val sdkRdd = { + val readOidcCredentials = OAuth2.ClientCredentials( + tokenUri = uri"https://login.microsoftonline.com/fake-tenant/oauth2/v2.0/token", + clientId = "fake_client_id", + clientSecret = "fake_secret", + scopes = List("https://api.cognitedata.com/.default"), + cdfProjectName = "fake_project" + ) val relationConfig = getDefaultConfig( CdfSparkAuth.OAuth2ClientCredentials(readOidcCredentials)(implicitBackend), readOidcCredentials.cdfProjectName diff --git a/src/test/scala/cognite/spark/v1/SparkTest.scala b/src/test/scala/cognite/spark/v1/SparkTest.scala index 836c73cc8..27557d85b 100644 --- a/src/test/scala/cognite/spark/v1/SparkTest.scala +++ b/src/test/scala/cognite/spark/v1/SparkTest.scala @@ -40,9 +40,10 @@ trait SparkTest { val clientId = sys.env("TEST_CLIENT_ID_BLUEFIELD") val clientSecret = sys.env("TEST_CLIENT_SECRET_BLUEFIELD") private val aadTenant = sys.env("TEST_AAD_TENANT_BLUEFIELD") + val project = sys.env("TEST_PROJECT") + val cluster = sys.env("TEST_CLUSTER") val tokenUri = s"https://login.microsoftonline.com/$aadTenant/oauth2/v2.0/token" - val project = "jetfiretest2" - val scopes = "https://api.cognitedata.com/.default" + val scopes = s"https://$cluster/.default" } val writeCredentials = OAuth2.ClientCredentials( @@ -59,7 +60,7 @@ trait SparkTest { val writeClient: GenericClient[IO] = new GenericClient( applicationName = "jetfire-test", projectName = writeCredentials.cdfProjectName, - baseUrl = s"https://api.cognitedata.com", + baseUrl = s"https://${OIDCWrite.cluster}", authProvider = writeAuthProvider, apiVersion = None, clientTag = None, @@ -84,42 +85,15 @@ trait SparkTest { .option("scopes", OIDCWrite.scopes) } - private val readClientId = System.getenv("TEST_OIDC_READ_CLIENT_ID") - // readClientSecret has to be renewed every 180 days at https://hub.cognite.com/open-industrial-data-211 - private val readClientSecret = System.getenv("TEST_OIDC_READ_CLIENT_SECRET") - private val readAadTenant = System.getenv("TEST_OIDC_READ_TENANT") - - assert( - readClientId != null && !readClientId.isEmpty, - "Environment variable \"TEST_OIDC_READ_CLIENT_ID\" was not set") - assert( - readClientSecret != null && !readClientSecret.isEmpty, - "Environment variable \"TEST_OIDC_READ_CLIENT_SECRET\" was not set") - assert( - readAadTenant != null && !readAadTenant.isEmpty, - "Environment variable \"TEST_OIDC_READ_TENANT\" was not set") - - private val readTokenUri = s"https://login.microsoftonline.com/$readAadTenant/oauth2/v2.0/token" - - val readOidcCredentials = OAuth2.ClientCredentials( - tokenUri = uri"$readTokenUri", - clientId = readClientId, - clientSecret = readClientSecret, - scopes = List("https://api.cognitedata.com/.default"), - cdfProjectName = "publicdata" - ) - def dataFrameReaderUsingOidc: DataFrameReader = spark.read .format("cognite.spark.v1") - .option("tokenUri", readTokenUri) - .option("clientId", readClientId) - .option("clientSecret", readClientSecret) - .option("project", "publicdata") - .option("scopes", "https://api.cognitedata.com/.default") - - // not needed to run tests, only for replicating some problems specific to this tenant - lazy val jetfiretest2ApiKey = System.getenv("TEST_APU_KEY_JETFIRETEST2") + .option("tokenUri", OIDCWrite.tokenUri) + .option("clientId", OIDCWrite.clientId) + .option("clientSecret", OIDCWrite.clientSecret) + .option("project", OIDCWrite.project) + .option("scopes", OIDCWrite.scopes) + val testDataSetId = 86163806167772L From 0301c7c897ac5e1478cd725e9eece67dfef2f5fb Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Mon, 7 Aug 2023 13:06:26 +0200 Subject: [PATCH 5/8] Update .github/workflows/main.yaml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 26616afbb..b44412184 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -42,7 +42,7 @@ jobs: TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} TEST_PROJECT: "spark-datasource-bluefield-tests" - TEST_CLUSTER: "bluefield.cognitedata.com" + TEST_CLUSTER: "bluefield.cognitedata.com" run: | TEST="test +Test/compile" if [ ${{github.ref }} == "refs/heads/master" ]; then From 76a87fcaf3b1e71e9add22d28ce688e4d38ef392 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Mon, 7 Aug 2023 20:57:38 +0200 Subject: [PATCH 6/8] Update .github/workflows/main.yaml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b44412184..7ea48b452 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -41,7 +41,7 @@ jobs: TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} - TEST_PROJECT: "spark-datasource-bluefield-tests" + TEST_PROJECT: "spark-datasource-bluefield-tests" TEST_CLUSTER: "bluefield.cognitedata.com" run: | TEST="test +Test/compile" From eae1b2d6c74f03e8dee0d11ace5dbad5ea1bfc2b Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Tue, 8 Aug 2023 13:59:38 +0200 Subject: [PATCH 7/8] Update .github/workflows/main.yaml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7ea48b452..b44412184 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -41,7 +41,7 @@ jobs: TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID_2 }} TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} - TEST_PROJECT: "spark-datasource-bluefield-tests" + TEST_PROJECT: "spark-datasource-bluefield-tests" TEST_CLUSTER: "bluefield.cognitedata.com" run: | TEST="test +Test/compile" From 70e2c02a886b5a2b405ec63f0935fdd8c25cc468 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Tue, 8 Aug 2023 14:19:05 +0200 Subject: [PATCH 8/8] Update .github/workflows/main.yaml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b44412184..26616afbb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -42,7 +42,7 @@ jobs: TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET_2 }} TEST_AAD_TENANT_BLUEFIELD: ${{ secrets.TEST_AAD_TENANT_BLUEFIELD_2 }} TEST_PROJECT: "spark-datasource-bluefield-tests" - TEST_CLUSTER: "bluefield.cognitedata.com" + TEST_CLUSTER: "bluefield.cognitedata.com" run: | TEST="test +Test/compile" if [ ${{github.ref }} == "refs/heads/master" ]; then