From 9a742fde820eaaf9d185dd662f0095cd820e9898 Mon Sep 17 00:00:00 2001 From: Dilyan Damyanov Date: Mon, 13 Jun 2022 16:37:01 +0100 Subject: [PATCH] Common: add tests for required property that is explicitly set to null (close #628) --- .../enrichments/TransformSpec.scala | 26 +++++++++++++++ .../utils/IgluUtilsSpec.scala | 32 +++++++++++-------- 2 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/TransformSpec.scala diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/TransformSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/TransformSpec.scala new file mode 100644 index 000000000..340f07b0e --- /dev/null +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/TransformSpec.scala @@ -0,0 +1,26 @@ +package com.snowplowanalytics.snowplow.enrich.common.enrichments + +import cats.syntax.option._ +import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent +import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers.MapOps +import org.specs2.mutable.Specification + +class TransformSpec extends Specification { + "transform should not drop properties with explicit null values" >> { + val ApplicationJsonWithCapitalCharset = "application/json; charset=UTF-8" + val raw = RawEvent( + CollectorPayload.Api("com.snowplowanalytics.snowplow", "tp2"), + Map("tv" -> "0", "p" -> "web", "e" -> "pv", "nuid" -> "123", "cx" -> "ewogICJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY29udGV4dHMvanNvbnNjaGVtYS8xLTAtMCIsCiAgImRhdGEiOiBbCiAgICB7CiAgICAgICJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY2xpZW50X3Nlc3Npb24vanNvbnNjaGVtYS8xLTAtMSIsCiAgICAgICJkYXRhIjogewogICAgICAgICJzZXNzaW9uSW5kZXgiOiAxLAogICAgICAgICJzdG9yYWdlTWVjaGFuaXNtIjogIkxPQ0FMX1NUT1JBR0UiLAogICAgICAgICJmaXJzdEV2ZW50SWQiOiAiNWMzM2ZjY2YtNmJlNS00Y2U2LWFmYjEtZTM0MDI2YTNjYTc1IiwKICAgICAgICAic2Vzc2lvbklkIjogIjIxYzJhMGRkLTg5MmQtNDJkMS1iMTU2LTNhOWQ0ZTE0N2VlZiIsCiAgICAgICAgInByZXZpb3VzU2Vzc2lvbklkIjogbnVsbCwKICAgICAgICAidXNlcklkIjogIjIwZDYzMWI4LTc4MzctNDlkZi1hNzNlLTZkYTczMTU0ZTZmZCIKICAgICAgfQogICAgfQogIF0KfQ==").toOpt, + ApplicationJsonWithCapitalCharset.some, + CollectorPayload.Source("source", "UTF-8", None), + CollectorPayload.Context(None, None, None, None, Nil, None) + ) + + val enriched = new EnrichedEvent + Transform.transform(raw, enriched) + + enriched.contexts mustEqual """{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-1","data":{"sessionIndex":1,"storageMechanism":"LOCAL_STORAGE","firstEventId":"5c33fccf-6be5-4ce6-afb1-e34026a3ca75","sessionId":"21c2a0dd-892d-42d1-b156-3a9d4e147eef","previousSessionId":null,"userId":"20d631b8-7837-49df-a73e-6da73154e6fd"}}]}""" + } +} diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala index 6d697e0a1..82bf8b8fc 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala @@ -14,16 +14,12 @@ package com.snowplowanalytics.snowplow.enrich.common.utils import org.specs2.mutable.Specification import org.specs2.matcher.ValidatedMatchers - import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} import com.snowplowanalytics.iglu.client.ClientError.{ResolutionError, ValidationError} - import com.snowplowanalytics.snowplow.badrows._ - import io.circe.Json - +import io.circe.parser._ import cats.data.NonEmptyList - import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers import com.snowplowanalytics.snowplow.enrich.common.utils.Clock._ @@ -92,16 +88,17 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers { "emailAddress": "hello@world.com" } }""" + val clientSessionData = s"""{ + "sessionIndex": 1, + "storageMechanism": "LOCAL_STORAGE", + "firstEventId": "5c33fccf-6be5-4ce6-afb1-e34026a3ca75", + "sessionId": "21c2a0dd-892d-42d1-b156-3a9d4e147eef", + "previousSessionId": null, + "userId": "20d631b8-7837-49df-a73e-6da73154e6fd" + }""" val clientSession = s"""{ "schema": "${clientSessionSchema.toSchemaUri}", - "data": { - "sessionIndex": 1, - "storageMechanism": "LOCAL_STORAGE", - "firstEventId": "5c33fccf-6be5-4ce6-afb1-e34026a3ca75", - "sessionId": "21c2a0dd-892d-42d1-b156-3a9d4e147eef", - "previousSessionId": null, - "userId": "20d631b8-7837-49df-a73e-6da73154e6fd" - } + "data": $clientSessionData }""" val noSchema = """{"schema":"iglu:com.snowplowanalytics.snowplow/foo/jsonschema/1-0-0", "data": {}}""" @@ -515,6 +512,15 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers { ) } } + + "return the extracted input context for an input that has a required property set to null if the schema explicitly allows it" >> { + val input = new EnrichedEvent + input.setContexts(buildInputContexts(List(clientSession))) + + val csd = parse(clientSessionData).getOrElse("") + + IgluUtils.extractAndValidateInputJsons(input, SpecHelpers.client, raw, processor).value must beRight((List(SelfDescribingData(clientSessionSchema, csd)), None)) + } } def buildUnstruct(sdj: String) =