Source MongoDb: fix parsing of DBRef objects (airbytehq#10276)

navi86 · Feb 22, 2022 · fbad2b8 · fbad2b8
1 parent c807d9c
commit fbad2b8
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 4 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -474,7 +474,7 @@
 - name: MongoDb
   sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e
   dockerRepository: airbyte/source-mongodb-v2
-  dockerImageTag: 0.1.11
+  dockerImageTag: 0.1.13
   documentationUrl: https://docs.airbyte.io/integrations/sources/mongodb-v2
   icon: mongodb.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4880,7 +4880,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-mongodb-v2:0.1.11"
+- dockerImage: "airbyte/source-mongodb-v2:0.1.13"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2"

diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java
@@ -4,14 +4,17 @@
 
 package io.airbyte.db.mongodb;
 
+import static java.util.Arrays.asList;
 import static org.bson.BsonType.ARRAY;
 import static org.bson.BsonType.DOCUMENT;
+import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.api.client.util.DateTime;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import com.mongodb.DBRefCodecProvider;
 import com.mongodb.client.AggregateIterable;
 import com.mongodb.client.MongoCollection;
 import io.airbyte.commons.json.Jsons;
@@ -36,6 +39,9 @@
 import org.bson.BsonTimestamp;
 import org.bson.BsonType;
 import org.bson.Document;
+import org.bson.codecs.*;
+import org.bson.codecs.configuration.CodecRegistry;
+import org.bson.codecs.jsr310.Jsr310CodecProvider;
 import org.bson.types.Decimal128;
 import org.bson.types.ObjectId;
 import org.bson.types.Symbol;
@@ -263,7 +269,20 @@ private static BsonType getBsonTypeByTypeAlias(String typeAlias) {
 
   private static BsonDocument toBsonDocument(final Document document) {
     try {
-      return document.toBsonDocument();
+      CodecRegistry customCodecRegistry =
+          fromProviders(asList(
+              new ValueCodecProvider(),
+              new BsonValueCodecProvider(),
+              new DocumentCodecProvider(),
+              new IterableCodecProvider(),
+              new MapCodecProvider(),
+              new Jsr310CodecProvider(),
+              new JsonObjectCodecProvider(),
+              new BsonCodecProvider(),
+              new DBRefCodecProvider()));
+
+      // Override the default codec registry
+      return document.toBsonDocument(BsonDocument.class, customCodecRegistry);
     } catch (final Exception e) {
       LOGGER.error("Exception while converting Document to BsonDocument: {}", e.getMessage());
       throw new RuntimeException(e);

diff --git a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-mongodb-v2
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=0.1.13
 LABEL io.airbyte.name=airbyte/source-mongodb-v2
diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md
@@ -102,6 +102,7 @@ For more information regarding configuration parameters, please see [MongoDb Doc
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.13  | 2022-02-21 | [10276](https://github.com/airbytehq/airbyte/pull/10276) | Create a custom codec registry to handle DBRef MongoDB objects |
 | 0.1.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option |
 | 0.1.11 | 2022-01-10 | [9238](https://github.com/airbytehq/airbyte/pull/9238) | Return only those collections for which the user has privileges |
 | 0.1.10 | 2021-12-30 | [9202](https://github.com/airbytehq/airbyte/pull/9202) | Update connector fields title/description |