fix: elasticsearch to bigtable test

GoogleCloudPlatform · Oct 23, 2024 · e5db06b · e5db06b
1 parent f30ea80
commit e5db06b
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,4 +24,6 @@ dist
 pipeline.json
 .ipynb_checkpoints/
 .venv/
-metastore_db/
+metastore_db/
+python/elastic.jks
+python/tls.cert
diff --git a/python/test/elasticsearch/test_elasticsearch_to_bigtable.py b/python/test/elasticsearch/test_elasticsearch_to_bigtable.py
@@ -35,16 +35,16 @@ def test_parse_args(self):
              "--es.bt.input.index=demo",
              "--es.bt.input.user=demo",
              "--es.bt.input.password=demo",
-             "--es.bt.project.id=GCP_PROJECT",
-             "--es.bt.instance.id=BIGTABLE_INSTANCE_ID",
+             "--spark.bigtable.project.id=GCP_PROJECT",
+             "--spark.bigtable.instance.id=BIGTABLE_INSTANCE_ID",
              "--es.bt.catalog.json={key:value}"])
 
         assert parsed_args["es.bt.input.node"] == "xxxxxxxxxxxx.us-central1.gcp.cloud.es.io:9243"
         assert parsed_args["es.bt.input.index"] == "demo"
         assert parsed_args["es.bt.input.user"] == "demo"
         assert parsed_args["es.bt.input.password"] == "demo"
-        assert parsed_args["es.bt.project.id"] == "GCP_PROJECT"
-        assert parsed_args["es.bt.instance.id"] == "BIGTABLE_INSTANCE_ID"
+        assert parsed_args["spark.bigtable.project.id"] == "GCP_PROJECT"
+        assert parsed_args["spark.bigtable.instance.id"] == "BIGTABLE_INSTANCE_ID"
         assert parsed_args["es.bt.catalog.json"] == '{key:value}'
 
     @mock.patch.object(pyspark.sql, 'SparkSession')
@@ -58,8 +58,8 @@ def test_run(self, mock_rename_columns, mock_spark_session):
              "--es.bt.input.index=demo",
              "--es.bt.input.user=demo",
              "--es.bt.input.password=demo",
-              "--es.bt.project.id=GCP_PROJECT",
-             "--es.bt.instance.id=BIGTABLE_INSTANCE_ID",
+             "--spark.bigtable.project.id=GCP_PROJECT",
+             "--spark.bigtable.instance.id=BIGTABLE_INSTANCE_ID",
              "--es.bt.catalog.json={key:value}"])
 
         mock_spark_session.sparkContext.newAPIHadoopRDD.return_value = mock_spark_session.rdd.RDD