MongoDb Source: Increase performance of discover (airbytehq#17614)

* MongoDb Source: Increase performance of discover * bump version * fixed tests * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <[email protected]>
navi86 · Oct 7, 2022 · 84f0299 · 84f0299
1 parent 0bec940
commit 84f0299
Show file tree

Hide file tree

Showing 8 changed files with 10 additions and 8 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -662,7 +662,7 @@
 - name: MongoDb
   sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e
   dockerRepository: airbyte/source-mongodb-v2
-  dockerImageTag: 0.1.18
+  dockerImageTag: 0.1.19
   documentationUrl: https://docs.airbyte.io/integrations/sources/mongodb-v2
   icon: mongodb.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -6801,7 +6801,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-mongodb-v2:0.1.18"
+- dockerImage: "airbyte/source-mongodb-v2:0.1.19"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2"

diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-mongodb-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.11
+LABEL io.airbyte.version=0.1.19
 LABEL io.airbyte.name=airbyte/source-mongodb-strict-encrypt
diff --git a/...java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java b/...java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java
@@ -40,7 +40,7 @@
 public class MongodbSourceStrictEncryptAcceptanceTest extends SourceAcceptanceTest {
 
   private static final String DATABASE_NAME = "test";
-  private static final String COLLECTION_NAME = "acceptance_test";
+  private static final String COLLECTION_NAME = "acceptance_test1";
   private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json");
   private static final String INSTANCE_TYPE = "instance_type";
 

diff --git a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-mongodb-v2
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.18
+LABEL io.airbyte.version=0.1.19
 LABEL io.airbyte.name=airbyte/source-mongodb-v2
diff --git a/...source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java b/...source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java
@@ -100,7 +100,8 @@ protected List<TableInfo<CommonField<BsonType>>> discoverInternal(final MongoDat
       throws Exception {
     final List<TableInfo<CommonField<BsonType>>> tableInfos = new ArrayList<>();
 
-    for (final String collectionName : getAuthorizedCollections(database)) {
+    final Set<String> authorizedCollections = getAuthorizedCollections(database);
+    authorizedCollections.parallelStream().forEach(collectionName -> {
       final MongoCollection<Document> collection = database.getCollection(collectionName);
       final List<CommonField<BsonType>> fields = MongoUtils.getUniqueFields(collection).stream().map(MongoUtils::nodeToCommonField).toList();
 
@@ -113,7 +114,7 @@ protected List<TableInfo<CommonField<BsonType>>> discoverInternal(final MongoDat
           .build();
 
       tableInfos.add(tableInfo);
-    }
+    });
     return tableInfos;
   }
 

diff --git a/...ntegrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java b/...ntegrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java
@@ -24,7 +24,7 @@
 public abstract class MongoDbSourceAbstractAcceptanceTest extends SourceAcceptanceTest {
 
   protected static final String DATABASE_NAME = "test";
-  protected static final String COLLECTION_NAME = "acceptance_test";
+  protected static final String COLLECTION_NAME = "acceptance_test1";
 
   protected JsonNode config;
   protected MongoDatabase database;

diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md
@@ -102,6 +102,7 @@ For more information regarding configuration parameters, please see [MongoDb Doc
 
 | Version | Date       | Pull Request | Subject                                                                                                   |
 |:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------------|
+| 0.1.19  | 2022-10-07 | [17614](https://github.com/airbytehq/airbyte/pull/17614) | Increased discover performance                             |
 | 0.1.18  | 2022-10-05 | [17590](https://github.com/airbytehq/airbyte/pull/17590) | Add ability to enforce SSL in MongoDB connector and check logic _                                         |
 | 0.1.17  | 2022-09-08 | [16401](https://github.com/airbytehq/airbyte/pull/16401) | Fixed bug with empty strings in fields with __aibyte_transform_                                           |
 | 0.1.16  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |