From b7f5f3b17974399037cf3a0c34a305279d1d30ec Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Fri, 3 Jun 2022 11:52:55 +0200 Subject: [PATCH 1/9] added compression feature for circe-akka-serializer --- README.md | 1 + .../src/main/resources/reference.conf | 14 +++++ .../ash/circe/CirceAkkaSerializer.scala | 25 ++++++-- .../virtuslab/ash/circe/CirceTraitCodec.scala | 44 ++++++++----- .../org/virtuslab/ash/circe/Compression.scala | 61 +++++++++++++++++++ .../src/test/resources/application.conf | 24 ++++++-- .../resources/more_than_1KiB_object_file.txt | 30 +++++++++ .../ash/circe/CirceAkkaSerializerSpec.scala | 43 ++++++++++++- .../ash/circe/CustomSerializer.scala | 5 +- ...a => MigrationAndCompressionTestKit.scala} | 8 +-- 10 files changed, 221 insertions(+), 34 deletions(-) create mode 100644 circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala create mode 100644 circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt rename circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/{MigrationTestKit.scala => MigrationAndCompressionTestKit.scala} (80%) diff --git a/README.md b/README.md index eef6bb02..662c43dc 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,7 @@ class ExampleSerializer(actorSystem: ExtendedActorSystem) override lazy val packagePrefix = "org.project" } ``` +`CirceAkkaSerializer` can be configured to use Gzip compression when serializing payloads greater than defined size (default is without compression). See [default reference.conf file](circe-akka-serializer/src/main/resources/reference.conf) with comments for details. For more guidelines on how to use the serializer, read [Akka documentation about serialization](https://doc.akka.io/docs/akka/current/serialization.html), diff --git a/circe-akka-serializer/src/main/resources/reference.conf b/circe-akka-serializer/src/main/resources/reference.conf index 4b0e3552..d256b384 100644 --- a/circe-akka-serializer/src/main/resources/reference.conf +++ b/circe-akka-serializer/src/main/resources/reference.conf @@ -1,5 +1,19 @@ # Default configuration org.virtuslab.ash { + verbose-debug-logging = off + + # settings for compression of the payload + compression { + # Compression algorithm. + # - off : no compression + # - gzip : using common java gzip + algorithm = off + + # If compression is enabled with the `algorithm` setting, + # the payload will be compressed if it's size is bigger than this value. + compress-larger-than = 0 KiB + } + } diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala index 4f5446d8..7e0e6e97 100644 --- a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala @@ -47,6 +47,16 @@ abstract class CirceAkkaSerializer[Ser <: AnyRef: ClassTag](system: ExtendedActo private lazy val log = Logging(system, getClass) private lazy val conf = system.settings.config.getConfig("org.virtuslab.ash") private lazy val isDebugEnabled = conf.getBoolean("verbose-debug-logging") && log.isDebugEnabled + private lazy val compressionAlgorithm: Compression.Algorithm = conf.getString("compression.algorithm") match { + case "off" => + Compression.Off + case "gzip" => + Compression.GZip(conf.getBytes("compression.compress-larger-than")) + case other => + throw new IllegalArgumentException( + s"Unknown compression algorithm value: [$other], possible values are: 'off' and 'gzip'") + } + protected val bufferSize: Int = 1024 * 4 override lazy val classTagEvidence: ClassTag[Ser] = implicitly[ClassTag[Ser]] override lazy val errorCallback: String => Unit = x => log.error(x) @@ -58,9 +68,10 @@ abstract class CirceAkkaSerializer[Ser <: AnyRef: ClassTag](system: ExtendedActo val startTime = if (isDebugEnabled) System.nanoTime else 0L codecsMap.get(manifest(o)) match { case Some((encoder, _)) => - val res = printer.print(encoder.asInstanceOf[Encoder[AnyRef]](o)).getBytes(UTF_8) - logDuration("Serialization", o, startTime, res) - res + val bytes = printer.print(encoder.asInstanceOf[Encoder[AnyRef]](o)).getBytes(UTF_8) + val result = Compression.compressIfNeeded(bytes, bufferSize, compressionAlgorithm) + logDuration("Serialization", o, startTime, result) + result case None => throw new RuntimeException( s"Serialization of [${o.getClass.getName}] failed. Call Register[A] for this class or its supertype and append result to `def codecs`.") @@ -71,9 +82,10 @@ abstract class CirceAkkaSerializer[Ser <: AnyRef: ClassTag](system: ExtendedActo val startTime = if (isDebugEnabled) System.nanoTime else 0L codecsMap.get(manifestMigrationsMap.getOrElse(manifest, manifest)) match { case Some((_, decoder)) => - val res = parser.parseByteArray(bytes).flatMap(_.as(decoder)).fold(e => throw e, identity) - logDuration("Deserialization", res, startTime, bytes) - res + val decompressedBytes = Compression.decompressIfNeeded(bytes, bufferSize) + val result = parser.parseByteArray(decompressedBytes).flatMap(_.as(decoder)).fold(e => throw e, identity) + logDuration("Deserialization", result, startTime, bytes) + result case None => throw new NotSerializableException( s"Manifest [$manifest] did not match any known codec. If you're not currently performing a rolling upgrade, you must add a manifest migration to correct codec.") @@ -115,4 +127,5 @@ abstract class CirceAkkaSerializer[Ser <: AnyRef: ClassTag](system: ExtendedActo bytes.length) } } + } diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala index 47af8757..95e8432a 100644 --- a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala @@ -54,15 +54,6 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { val errorCallback: String => Unit - Seq( - (codecs, "codecs"), - (manifestMigrations, "manifestMigrations"), - (packagePrefix, "packagePrefix"), - (classTagEvidence, "classTagEvidence"), - (errorCallback, "errorCallback")).foreach { x => - assert(x._1 != null, s"${x._2} must be declared as a def or a lazy val to work correctly") - } - private val mirror = ru.runtimeMirror(getClass.getClassLoader) protected val codecsMap: Map[String, (Encoder[_ <: Ser], Decoder[_ <: Ser])] = codecs @@ -85,8 +76,7 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { .toMap .withDefaultValue("") - def manifest(o: AnyRef): String = parentsUpToRegisteredTypeMap(o.getClass.getName) - + // Decoder apply method - decodes from Json into an object of type Ser override def apply(c: HCursor): Result[Ser] = { c.value.asObject match { case Some(obj) => @@ -103,6 +93,7 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { } } + // Encoder apply method - encodes given object of type Ser into Json override def apply(a: Ser): Json = { val manifestString = manifest(a) val encoder = codecsMap.get(manifestString) match { @@ -114,6 +105,33 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { Json.obj((manifestString, encoder.asInstanceOf[Encoder[Ser]](a))) } + def manifest(o: AnyRef): String = parentsUpToRegisteredTypeMap(o.getClass.getName) + + /* + * All code below serves as a check - it checks, + * whether class extending this trait is a valid implementation. + * doNeededChecksOnStart() gets invoked on object creation. + */ + doNeededChecksOnStart() + + private def doNeededChecksOnStart(): Unit = { + checkImplementationForInvalidMembersDeclarations() + checkSerializableTypesForMissingCodec(packagePrefix) + checkCodecsForNull() + checkCodecsForDuplication() + } + + private def checkImplementationForInvalidMembersDeclarations(): Unit = { + Seq( + (codecs, "codecs"), + (manifestMigrations, "manifestMigrations"), + (packagePrefix, "packagePrefix"), + (classTagEvidence, "classTagEvidence"), + (errorCallback, "errorCallback")).foreach { x => + assert(x._1 != null, s"${x._2} must be declared as a def or a lazy val to work correctly") + } + } + private def checkSerializableTypesForMissingCodec(packagePrefix: String): Unit = { val reflections = new Reflections(packagePrefix) val foundSerializables = reflections.getSubTypesOf(classTag[Ser].runtimeClass).asScala.filterNot(_.isInterface) @@ -142,8 +160,4 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { errorCallback(s"Codec for class ${x._1} has been declared multiple times with types ${x._2.mkString(",")}.") } } - - checkSerializableTypesForMissingCodec(packagePrefix) - checkCodecsForNull() - checkCodecsForDuplication() } diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala new file mode 100644 index 00000000..0bfc93a2 --- /dev/null +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala @@ -0,0 +1,61 @@ +package org.virtuslab.ash.circe + +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream + +import scala.annotation.tailrec + +object Compression { + sealed trait Algorithm + case object Off extends Algorithm + case class GZip(greaterThan: Long) extends Algorithm + // TODO: add support for LZ4 java compression: + // https://github.com/VirtusLab/akka-serialization-helper/issues/159 + // case class LZ4(greaterThan: Long) extends Algorithm + + private[circe] def compressIfNeeded( + bytes: Array[Byte], + bufferSize: Int, + compressionAlgorithm: Algorithm): Array[Byte] = + compressionAlgorithm match { + case Compression.Off => bytes + case Compression.GZip(largerThan) => + if (bytes.length <= largerThan) { + bytes + } else { + val byteArrayOutputStream = new ByteArrayOutputStream(bufferSize) + val outputStream = new GZIPOutputStream(byteArrayOutputStream) + try outputStream.write(bytes) + finally outputStream.close() + byteArrayOutputStream.toByteArray + } + } + + private[circe] def decompressIfNeeded(bytes: Array[Byte], bufferSize: Int): Array[Byte] = + if (isCompressedWithGzip(bytes)) { + val inputStream = new GZIPInputStream(new ByteArrayInputStream(bytes)) + val outputStream = new ByteArrayOutputStream() + val buffer = new Array[Byte](bufferSize) + + @tailrec def readChunk(): Unit = + inputStream.read(buffer) match { + case -1 => () + case n => + outputStream.write(buffer, 0, n) + readChunk() + } + + try readChunk() + finally inputStream.close() + outputStream.toByteArray + } else { + bytes + } + + private[circe] def isCompressedWithGzip(bytes: Array[Byte]): Boolean = + (bytes != null) && (bytes.length >= 2) && + (bytes(0) == GZIPInputStream.GZIP_MAGIC.toByte) && + (bytes(1) == (GZIPInputStream.GZIP_MAGIC >> 8).toByte) +} diff --git a/circe-akka-serializer/src/test/resources/application.conf b/circe-akka-serializer/src/test/resources/application.conf index a7070e43..5a5ecb81 100644 --- a/circe-akka-serializer/src/test/resources/application.conf +++ b/circe-akka-serializer/src/test/resources/application.conf @@ -9,9 +9,23 @@ akka.actor { allow-java-serialization = off } -#Uncomment below to enable logging - +# Uncomment chosen settings below to enable debug logging OR change compression settings #akka.loglevel = "DEBUG" -#org.virtuslab.ash { -# verbose-debug-logging = on -#} +org.virtuslab.ash { + + # enables debug logging + #verbose-debug-logging = on + + # settings for compression of the payload + compression { + # Compression algorithm. + # - off : no compression + # - gzip : using common java gzip + algorithm = off + + # If compression is enabled with the `algorithm` setting, + # the payload will be compressed if it's size is bigger than this value. + compress-larger-than = 1 KiB # value set for testing purposes - do not change + } + +} diff --git a/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt b/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt new file mode 100644 index 00000000..7cbac02e --- /dev/null +++ b/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt @@ -0,0 +1,30 @@ +type:: +type::phoenix.bets.infrastructure.BetsAkkaSerialization.type +type::AnyRef +type::phoenix.core.serialization.PhoenixAkkaSerialization[phoenix.bets.infrastructure.BetsAkkaSerializable] +type::phoenix.core.serialization.PhoenixCodecs +type::Unit +type::Object +type::(): Object +type::phoenix.bets.infrastructure.BetsAkkaSerialization.type +type::phoenix.bets.infrastructure.BetsAkkaSerialization.type +type::io.circe.Codec[phoenix.punters.PunterEntity.PunterId] +type::io.circe.Codec.AsObject[phoenix.punters.PunterEntity.PunterId] +type::(implicit codec: shapeless.Lazy[io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId]]): io.circe.Codec.AsObject[phoenix.punters.PunterEntity.PunterId] +type::[A](implicit codec: shapeless.Lazy[io.circe.generic.codec.DerivedAsObjectCodec[A]]): io.circe.Codec.AsObject[A] +type::io.circe.generic.semiauto.type +type::io.circe.generic.type +type::io.circe.type +type::io.type +type::phoenix.punters.PunterEntity.PunterId +type::shapeless.Lazy[io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId]] +type::io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId] +type::anon$lazy$macro$5 +type::Serializable +type::anon$lazy$macro$5.super.type +type::[T0]T0 +type::(implicit gen: shapeless.LabelledGeneric.Aux[phoenix.punters.PunterEntity.PunterId,shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out], codec: shapeless.Lazy[io.circe.generic.codec.ReprAsObjectCodec[shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out]]): io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId] +type::[A, R](implicit gen: shapeless.LabelledGeneric.Aux[A,R], codec: shapeless.Lazy[io.circe.generic.codec.ReprAsObjectCodec[R]]): io.circe.generic.codec.DerivedAsObjectCodec[A] +type::io.circe.generic.codec.DerivedAsObjectCodec.type +type::io.circe.generic.codec.type +type::shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out \ No newline at end of file diff --git a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala index 11fe1199..3ea7a7d0 100644 --- a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala +++ b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala @@ -1,13 +1,17 @@ package org.virtuslab.ash.circe +import scala.io.Source + import akka.actor.testkit.typed.scaladsl.ActorTestKit import akka.actor.testkit.typed.scaladsl.SerializationTestKit import akka.actor.typed.ActorSystem import com.typesafe.config.ConfigFactory +import com.typesafe.config.ConfigValueFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpecLike -import org.virtuslab.ash.circe.MigrationTestKit.SerializationData +import org.virtuslab.ash.circe.Compression.isCompressedWithGzip +import org.virtuslab.ash.circe.MigrationAndCompressionTestKit.SerializationData import org.virtuslab.ash.circe.data.Tree.Node._ import org.virtuslab.ash.circe.data._ @@ -17,7 +21,7 @@ class CirceAkkaSerializerSpec extends AnyWordSpecLike with should.Matchers { val testKit: ActorTestKit = ActorTestKit(config) val system: ActorSystem[Nothing] = testKit.system val serializationTestKit = new SerializationTestKit(system) - val migrationTestKit = new MigrationTestKit(system) + val migrationTestKit = new MigrationAndCompressionTestKit(system) "serialize standard data" in { import org.virtuslab.ash.circe.data.StdData._ @@ -83,5 +87,40 @@ class CirceAkkaSerializerSpec extends AnyWordSpecLike with should.Matchers { serializationTestKit.verifySerialization(GenericClass(StdData.One(1, 1, "aaa"), Tree.Leaf())) } + // setup for testing compression feature - with compression enabled + val compressionConfig = + config.withValue("org.virtuslab.ash.compression.algorithm", ConfigValueFactory.fromAnyRef("gzip")) + val compressionTestKit: ActorTestKit = ActorTestKit(compressionConfig) + val compressionActorSystem: ActorSystem[Nothing] = compressionTestKit.system + val compressionSerializationTestKit = new MigrationAndCompressionTestKit(compressionActorSystem) + + val heavyWeightString = Source.fromResource("more_than_1KiB_object_file.txt").getLines.toList.mkString("\n") + val lightWeightString = "x" + val largeSerializableObject = StdData.One(123, 456.0f, heavyWeightString) + val smallSerializableObject = StdData.One(123, 456.0f, lightWeightString) + + "compress payload using GZip compression algorithm when gzip compression is enabled and payload's size is bigger than threshold" in { + val largeObjectSerialized = compressionSerializationTestKit.serialize(largeSerializableObject) + isCompressedWithGzip(largeObjectSerialized._1) shouldBe true + } + + "decompress payload properly when it has been compressed previously with GZip compression algorithm" in { + val largeObjectSerialized = compressionSerializationTestKit.serialize(largeSerializableObject) + val deserializedObject = compressionSerializationTestKit.deserialize(largeObjectSerialized) + deserializedObject shouldEqual largeSerializableObject + } + + "not compress payload when gzip compression is enabled and payload's size is smaller than threshold" in { + val smallObjectSerialized = compressionSerializationTestKit.serialize(smallSerializableObject) + isCompressedWithGzip(smallObjectSerialized._1) shouldBe false + } + + // below example uses standard `config` from circe-akka-serializer/src/test/resources/application.conf + // which is available in the migrationTestKit object - so compression is disabled + "not compress payload when compression is configuration is set to 'off'" in { + val largeObjectSerialized = migrationTestKit.serialize(largeSerializableObject) + isCompressedWithGzip(largeObjectSerialized._1) shouldBe false + } + } } diff --git a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CustomSerializer.scala b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CustomSerializer.scala index b7b2c74e..3df6caa9 100644 --- a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CustomSerializer.scala +++ b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CustomSerializer.scala @@ -17,7 +17,7 @@ class CustomSerializer(actorSystem: ExtendedActorSystem) override def identifier: Int = 42352 - override lazy val codecs = + override lazy val codecs: Seq[Registration[_ <: CirceSerializabilityTrait]] = Seq( Register[Tree], Register[StdData], @@ -26,7 +26,8 @@ class CustomSerializer(actorSystem: ExtendedActorSystem) Register(implicitly[ru.TypeTag[ModifiedCodec]], prepareEncoder, prepareDecoder), Register[GenericClass[CirceSerializabilityTrait, CirceSerializabilityTrait]]) - override lazy val manifestMigrations = Seq("org.virtuslab.ash.data.OldName" -> classOf[TopTraitMigration]) + override lazy val manifestMigrations: Seq[(String, Class[TopTraitMigration])] = + Seq("org.virtuslab.ash.data.OldName" -> classOf[TopTraitMigration]) override lazy val packagePrefix = "org.virtuslab.ash" } diff --git a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/MigrationTestKit.scala b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/MigrationAndCompressionTestKit.scala similarity index 80% rename from circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/MigrationTestKit.scala rename to circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/MigrationAndCompressionTestKit.scala index 71d90cb5..fc008e17 100644 --- a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/MigrationTestKit.scala +++ b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/MigrationAndCompressionTestKit.scala @@ -5,10 +5,10 @@ import akka.actor.typed.scaladsl.adapter._ import akka.serialization.SerializationExtension import akka.serialization.Serializers -import org.virtuslab.ash.circe.MigrationTestKit.SerializationData +import org.virtuslab.ash.circe.MigrationAndCompressionTestKit.SerializationData /** - * Utilities to test serialization migration. + * Utilities to test serialization migration and compression. * To perform round trip: * {{{ * val migrationTestKit = new MigrationTestKit(system) @@ -18,7 +18,7 @@ import org.virtuslab.ash.circe.MigrationTestKit.SerializationData * original == result * }}} */ -class MigrationTestKit(system: ActorSystem[_]) { +class MigrationAndCompressionTestKit(system: ActorSystem[_]) { private val serialization = SerializationExtension(system.toClassic) def serialize(objAnyRef: AnyRef): SerializationData = { @@ -33,6 +33,6 @@ class MigrationTestKit(system: ActorSystem[_]) { } } -object MigrationTestKit { +object MigrationAndCompressionTestKit { type SerializationData = (Array[Byte], Int, String) } From cb9b6e69ec90cfad87178c61769546f8db5b073f Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Fri, 3 Jun 2022 12:17:23 +0200 Subject: [PATCH 2/9] Added newline character for checks --- .../src/test/resources/more_than_1KiB_object_file.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt b/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt index 7cbac02e..89e7e53f 100644 --- a/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt +++ b/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt @@ -27,4 +27,4 @@ type::(implicit gen: shapeless.LabelledGeneric.Aux[phoenix.punters.PunterEntity. type::[A, R](implicit gen: shapeless.LabelledGeneric.Aux[A,R], codec: shapeless.Lazy[io.circe.generic.codec.ReprAsObjectCodec[R]]): io.circe.generic.codec.DerivedAsObjectCodec[A] type::io.circe.generic.codec.DerivedAsObjectCodec.type type::io.circe.generic.codec.type -type::shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out \ No newline at end of file +type::shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out From b3b5152c625269e1ab2fae388582de7217bea92b Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Fri, 3 Jun 2022 12:21:00 +0200 Subject: [PATCH 3/9] removed trailing space from README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 662c43dc..b274067f 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ class ExampleSerializer(actorSystem: ExtendedActorSystem) override lazy val packagePrefix = "org.project" } ``` -`CirceAkkaSerializer` can be configured to use Gzip compression when serializing payloads greater than defined size (default is without compression). See [default reference.conf file](circe-akka-serializer/src/main/resources/reference.conf) with comments for details. +`CirceAkkaSerializer` can be configured to use Gzip compression when serializing payloads greater than defined size (default is without compression). See [default reference.conf file](circe-akka-serializer/src/main/resources/reference.conf) with comments for details. For more guidelines on how to use the serializer, read [Akka documentation about serialization](https://doc.akka.io/docs/akka/current/serialization.html), From be30f02aa652817c37f5997efcf53c4141561c32 Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Fri, 3 Jun 2022 12:30:42 +0200 Subject: [PATCH 4/9] few minor PR fixes --- .../src/main/scala/org/virtuslab/ash/circe/Compression.scala | 3 +-- .../org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala index 0bfc93a2..6239f0e2 100644 --- a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala @@ -11,8 +11,7 @@ object Compression { sealed trait Algorithm case object Off extends Algorithm case class GZip(greaterThan: Long) extends Algorithm - // TODO: add support for LZ4 java compression: - // https://github.com/VirtusLab/akka-serialization-helper/issues/159 + // TODO (#159): add support for LZ4 java compression // case class LZ4(greaterThan: Long) extends Algorithm private[circe] def compressIfNeeded( diff --git a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala index 3ea7a7d0..e08bb93f 100644 --- a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala +++ b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala @@ -94,7 +94,7 @@ class CirceAkkaSerializerSpec extends AnyWordSpecLike with should.Matchers { val compressionActorSystem: ActorSystem[Nothing] = compressionTestKit.system val compressionSerializationTestKit = new MigrationAndCompressionTestKit(compressionActorSystem) - val heavyWeightString = Source.fromResource("more_than_1KiB_object_file.txt").getLines.toList.mkString("\n") + val heavyWeightString = Source.fromResource("more_than_1KiB_object_file.txt").getLines().toList.mkString("\n") val lightWeightString = "x" val largeSerializableObject = StdData.One(123, 456.0f, heavyWeightString) val smallSerializableObject = StdData.One(123, 456.0f, lightWeightString) From 5a67a19fe69c998efc7a390166d451a7e1f63316 Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Fri, 3 Jun 2022 12:40:22 +0200 Subject: [PATCH 5/9] replace real logs with random strings --- .../resources/more_than_1KiB_object_file.txt | 93 +++++++++++++------ 1 file changed, 63 insertions(+), 30 deletions(-) diff --git a/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt b/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt index 89e7e53f..62b42176 100644 --- a/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt +++ b/circe-akka-serializer/src/test/resources/more_than_1KiB_object_file.txt @@ -1,30 +1,63 @@ -type:: -type::phoenix.bets.infrastructure.BetsAkkaSerialization.type -type::AnyRef -type::phoenix.core.serialization.PhoenixAkkaSerialization[phoenix.bets.infrastructure.BetsAkkaSerializable] -type::phoenix.core.serialization.PhoenixCodecs -type::Unit -type::Object -type::(): Object -type::phoenix.bets.infrastructure.BetsAkkaSerialization.type -type::phoenix.bets.infrastructure.BetsAkkaSerialization.type -type::io.circe.Codec[phoenix.punters.PunterEntity.PunterId] -type::io.circe.Codec.AsObject[phoenix.punters.PunterEntity.PunterId] -type::(implicit codec: shapeless.Lazy[io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId]]): io.circe.Codec.AsObject[phoenix.punters.PunterEntity.PunterId] -type::[A](implicit codec: shapeless.Lazy[io.circe.generic.codec.DerivedAsObjectCodec[A]]): io.circe.Codec.AsObject[A] -type::io.circe.generic.semiauto.type -type::io.circe.generic.type -type::io.circe.type -type::io.type -type::phoenix.punters.PunterEntity.PunterId -type::shapeless.Lazy[io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId]] -type::io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId] -type::anon$lazy$macro$5 -type::Serializable -type::anon$lazy$macro$5.super.type -type::[T0]T0 -type::(implicit gen: shapeless.LabelledGeneric.Aux[phoenix.punters.PunterEntity.PunterId,shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out], codec: shapeless.Lazy[io.circe.generic.codec.ReprAsObjectCodec[shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out]]): io.circe.generic.codec.DerivedAsObjectCodec[phoenix.punters.PunterEntity.PunterId] -type::[A, R](implicit gen: shapeless.LabelledGeneric.Aux[A,R], codec: shapeless.Lazy[io.circe.generic.codec.ReprAsObjectCodec[R]]): io.circe.generic.codec.DerivedAsObjectCodec[A] -type::io.circe.generic.codec.DerivedAsObjectCodec.type -type::io.circe.generic.codec.type -type::shapeless.labelled.FieldType[Symbol @@ String("value"),String] :: shapeless.ops.hlist.ZipWithKeys.hnilZipWithKeys.Out +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || +this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || this is some random string || From 9694e2d3b5c7e7284ad4694f386562878f17cb16 Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Fri, 3 Jun 2022 13:47:20 +0200 Subject: [PATCH 6/9] fix structure in circe-akka-serializer reference.conf --- .../src/main/resources/reference.conf | 24 ++++++++--------- .../ash/circe/CirceAkkaSerializer.scala | 2 +- .../src/test/resources/application.conf | 26 +++++++++---------- .../ash/circe/CirceAkkaSerializerSpec.scala | 2 +- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/circe-akka-serializer/src/main/resources/reference.conf b/circe-akka-serializer/src/main/resources/reference.conf index d256b384..ce838d7c 100644 --- a/circe-akka-serializer/src/main/resources/reference.conf +++ b/circe-akka-serializer/src/main/resources/reference.conf @@ -1,19 +1,19 @@ # Default configuration org.virtuslab.ash { + circe { + verbose-debug-logging = off - verbose-debug-logging = off + # settings for compression of the payload + compression { + # Compression algorithm. + # - off : no compression + # - gzip : using common java gzip + algorithm = off - # settings for compression of the payload - compression { - # Compression algorithm. - # - off : no compression - # - gzip : using common java gzip - algorithm = off - - # If compression is enabled with the `algorithm` setting, - # the payload will be compressed if it's size is bigger than this value. - compress-larger-than = 0 KiB + # If compression is enabled with the `algorithm` setting, + # the payload will be compressed if its size is bigger than this value. + compress-larger-than = 32 KiB + } } - } diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala index 7e0e6e97..437d8a65 100644 --- a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceAkkaSerializer.scala @@ -45,7 +45,7 @@ abstract class CirceAkkaSerializer[Ser <: AnyRef: ClassTag](system: ExtendedActo with AkkaCodecs { private lazy val log = Logging(system, getClass) - private lazy val conf = system.settings.config.getConfig("org.virtuslab.ash") + private lazy val conf = system.settings.config.getConfig("org.virtuslab.ash.circe") private lazy val isDebugEnabled = conf.getBoolean("verbose-debug-logging") && log.isDebugEnabled private lazy val compressionAlgorithm: Compression.Algorithm = conf.getString("compression.algorithm") match { case "off" => diff --git a/circe-akka-serializer/src/test/resources/application.conf b/circe-akka-serializer/src/test/resources/application.conf index 5a5ecb81..ea87a6b4 100644 --- a/circe-akka-serializer/src/test/resources/application.conf +++ b/circe-akka-serializer/src/test/resources/application.conf @@ -12,20 +12,20 @@ akka.actor { # Uncomment chosen settings below to enable debug logging OR change compression settings #akka.loglevel = "DEBUG" org.virtuslab.ash { + circe { + # enables debug logging + #verbose-debug-logging = on - # enables debug logging - #verbose-debug-logging = on + # settings for compression of the payload + compression { + # Compression algorithm. + # - off : no compression + # - gzip : using common java gzip + algorithm = off - # settings for compression of the payload - compression { - # Compression algorithm. - # - off : no compression - # - gzip : using common java gzip - algorithm = off - - # If compression is enabled with the `algorithm` setting, - # the payload will be compressed if it's size is bigger than this value. - compress-larger-than = 1 KiB # value set for testing purposes - do not change + # If compression is enabled with the `algorithm` setting, + # the payload will be compressed if it's size is bigger than this value. + compress-larger-than = 1 KiB # value set for testing purposes - do not change + } } - } diff --git a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala index e08bb93f..3c06da66 100644 --- a/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala +++ b/circe-akka-serializer/src/test/scala/org/virtuslab/ash/circe/CirceAkkaSerializerSpec.scala @@ -89,7 +89,7 @@ class CirceAkkaSerializerSpec extends AnyWordSpecLike with should.Matchers { // setup for testing compression feature - with compression enabled val compressionConfig = - config.withValue("org.virtuslab.ash.compression.algorithm", ConfigValueFactory.fromAnyRef("gzip")) + config.withValue("org.virtuslab.ash.circe.compression.algorithm", ConfigValueFactory.fromAnyRef("gzip")) val compressionTestKit: ActorTestKit = ActorTestKit(compressionConfig) val compressionActorSystem: ActorSystem[Nothing] = compressionTestKit.system val compressionSerializationTestKit = new MigrationAndCompressionTestKit(compressionActorSystem) From f15b0da430fc2e819585e67e9643c3ddaa6cd2f0 Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Fri, 3 Jun 2022 16:56:55 +0200 Subject: [PATCH 7/9] PR fixes and improvements --- .../org/virtuslab/ash/circe/CirceTraitCodec.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala index 95e8432a..4a927c11 100644 --- a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/CirceTraitCodec.scala @@ -76,7 +76,9 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { .toMap .withDefaultValue("") - // Decoder apply method - decodes from Json into an object of type Ser + /** + * Decoder apply method - decodes from Json into an object of type Ser + */ override def apply(c: HCursor): Result[Ser] = { c.value.asObject match { case Some(obj) => @@ -93,7 +95,9 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { } } - // Encoder apply method - encodes given object of type Ser into Json + /** + * Encoder apply method - encodes given object of type Ser into Json + */ override def apply(a: Ser): Json = { val manifestString = manifest(a) val encoder = codecsMap.get(manifestString) match { @@ -115,13 +119,13 @@ trait CirceTraitCodec[Ser <: AnyRef] extends Codec[Ser] { doNeededChecksOnStart() private def doNeededChecksOnStart(): Unit = { - checkImplementationForInvalidMembersDeclarations() + checkImplementationForInvalidMemberDeclarations() checkSerializableTypesForMissingCodec(packagePrefix) checkCodecsForNull() checkCodecsForDuplication() } - private def checkImplementationForInvalidMembersDeclarations(): Unit = { + private def checkImplementationForInvalidMemberDeclarations(): Unit = { Seq( (codecs, "codecs"), (manifestMigrations, "manifestMigrations"), From 053c11597d2feebb01929d5dda01b4635a8abb00 Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Sat, 4 Jun 2022 18:39:52 +0200 Subject: [PATCH 8/9] added useful comment on check --- .../org/virtuslab/ash/circe/Compression.scala | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala index 6239f0e2..8a918bb4 100644 --- a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala @@ -53,6 +53,20 @@ object Compression { bytes } + /* + Since we are encoding JSON for Ser <: AnyRef types - they will start with '{' char for Json objects or with '[' char for Arrays. + Thus, the first element of the `bytes` array is either the 123 Byte number - which is the decimal representation of the { character, + or - the 91 Byte number - which is the decimal representation of the [ character. + + So, below quick comment on why isCompressedWithGzip will not return false positives (for not compressed JSON data): + + bytes(0) == GZIPInputStream.GZIP_MAGIC.toByte + gets evaluated to: + bytes(0) == 35615.toByte + which gets evaluated to: + bytes(0) == 31 // where 31 is of type Byte + And since bytes(0) holds a Byte with value equal to 123 or 91 - this will never be true. + */ private[circe] def isCompressedWithGzip(bytes: Array[Byte]): Boolean = (bytes != null) && (bytes.length >= 2) && (bytes(0) == GZIPInputStream.GZIP_MAGIC.toByte) && From 718ff204198a42545a31753460f31d350bd09dee Mon Sep 17 00:00:00 2001 From: LukaszKontowski Date: Sat, 4 Jun 2022 19:04:32 +0200 Subject: [PATCH 9/9] added more comments --- .../src/main/resources/reference.conf | 6 ++++-- .../scala/org/virtuslab/ash/circe/Compression.scala | 13 +++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/circe-akka-serializer/src/main/resources/reference.conf b/circe-akka-serializer/src/main/resources/reference.conf index ce838d7c..c089231e 100644 --- a/circe-akka-serializer/src/main/resources/reference.conf +++ b/circe-akka-serializer/src/main/resources/reference.conf @@ -4,9 +4,11 @@ org.virtuslab.ash { circe { verbose-debug-logging = off - # settings for compression of the payload + # Settings for compression of the payload - here we decide, if compression should be possible + # when serializing an object. It does not not affect deserialization process - so even with + # `compression.algorithm = off` deserialization of objects compressed with Gzip will be successfull. compression { - # Compression algorithm. + # Compression algorithm: # - off : no compression # - gzip : using common java gzip algorithm = off diff --git a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala index 8a918bb4..720ea0b6 100644 --- a/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala +++ b/circe-akka-serializer/src/main/scala/org/virtuslab/ash/circe/Compression.scala @@ -54,9 +54,14 @@ object Compression { } /* - Since we are encoding JSON for Ser <: AnyRef types - they will start with '{' char for Json objects or with '[' char for Arrays. - Thus, the first element of the `bytes` array is either the 123 Byte number - which is the decimal representation of the { character, - or - the 91 Byte number - which is the decimal representation of the [ character. + Since we are encoding JSON for Ser <: AnyRef types - they can start with: + a) '{' char for Json objects or + b) '[' char for Arrays or + c) '"' char for String + Thus, the first element of the `bytes` array could be one of three below: + a) 123 Byte number - which is the decimal representation of the { character + b) 91 Byte number - which is the decimal representation of the [ character + c) 34 Byte number - which is the decimal representation of the " character So, below quick comment on why isCompressedWithGzip will not return false positives (for not compressed JSON data): @@ -65,7 +70,7 @@ object Compression { bytes(0) == 35615.toByte which gets evaluated to: bytes(0) == 31 // where 31 is of type Byte - And since bytes(0) holds a Byte with value equal to 123 or 91 - this will never be true. + And since bytes(0) holds a Byte with value equal to 123, 91 or 34 - this will never be true. */ private[circe] def isCompressedWithGzip(bytes: Array[Byte]): Boolean = (bytes != null) && (bytes.length >= 2) &&