From 56889c0fdcbc94b27c3a0973d3d4a0be4bc0015a Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 15:13:32 +0200 Subject: [PATCH 01/12] json array implicits for doobie --- .../integration/V1.2.15__actors_json_seq.ddl | 21 ++++ .../V1.2.16__insert_actors_json.sql | 23 +++++ .../V1.2.17__retrieve_actors_json.sql | 22 +++++ .../V1.2.18__retrieve_actors_jsonb.sql | 22 +++++ .../doobie/postgres/implicits/package.scala | 96 +++++++++++++++++++ .../absa/db/fadb/doobie/JsonArrayTests.scala | 75 +++++++++++++++ .../db/fadb/testing/classes/DoobieTest.scala | 6 ++ project/Dependencies.scala | 16 +++- 8 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl create mode 100644 demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql create mode 100644 demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql create mode 100644 demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql create mode 100644 doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala create mode 100644 doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala diff --git a/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl b/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl new file mode 100644 index 00000000..de4c6c91 --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl @@ -0,0 +1,21 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS integration.actors_json_seq ( + id SERIAL PRIMARY KEY, + actors_json JSON[], + actors_jsonb JSONB[] +); diff --git a/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql b/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql new file mode 100644 index 00000000..b59c5ab7 --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql @@ -0,0 +1,23 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.insert_actors_json(actorsJson JSON[], actorsJsonb JSONB[]) + RETURNS void AS $$ +BEGIN + INSERT INTO integration.actors_json_seq (actors_json, actors_jsonb) + VALUES (actorsJson, actorsJsonb); +END; +$$ LANGUAGE plpgsql; diff --git a/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql b/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql new file mode 100644 index 00000000..a34a472a --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql @@ -0,0 +1,22 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.retrieve_actors_json(idUntil INT) + RETURNS TABLE(actors_json JSON[]) AS $$ +BEGIN + RETURN QUERY SELECT a.actors_json FROM integration.actors_json_seq AS a WHERE id <= idUntil; +END; +$$ LANGUAGE plpgsql; diff --git a/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql b/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql new file mode 100644 index 00000000..bb5e3e4f --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql @@ -0,0 +1,22 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.retrieve_actors_jsonb(idUntil INT) + RETURNS TABLE(actors_jsonb JSONB[]) AS $$ +BEGIN + RETURN QUERY SELECT a.actors_jsonb FROM integration.actors_json_seq AS a WHERE id <= idUntil; +END; +$$ LANGUAGE plpgsql; diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala new file mode 100644 index 00000000..625d1d02 --- /dev/null +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala @@ -0,0 +1,96 @@ +package za.co.absa.db.fadb.doobie.postgres + +import cats.Show +import cats.data.NonEmptyList +import doobie.postgres.implicits._ +import doobie.{Get, Put} +import io.circe.{Json => CirceJson} +import org.postgresql.jdbc.PgArray +import org.postgresql.util.PGobject +import io.circe.parser._ + +import scala.util.Try + +package object implicits { + + private implicit val showPgArray: Show[PgArray] = Show.fromToString + + implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] + .tmap(map => map.map { case (k, v) => k -> Option(v) }) + + private def circeJsonListToPGJsonArrayString(jsonList: List[CirceJson]): String = { + val arrayElements = jsonList.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s""""$escapedJsonString"""" + } + + arrayElements.mkString("{", ",", "}") + } + + private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[CirceJson]] = { + Try { + Option(pgArray.getArray) match { + case Some(array: Array[_]) => array.collect { + case str: String => parse(str).toTry.get + case other => parse(other.toString).toTry.get + }.toList + case None => List.empty[CirceJson] + case _ => throw new IllegalArgumentException("Unexpected type encountered.") + } + } + .toEither + .left.map(_.getMessage) + } + + object Sequence { + + implicit val get: Get[Seq[String]] = Get[List[String]].map(_.toSeq) + implicit val put: Put[Seq[String]] = Put[List[String]].contramap(_.toList) + + } + + object Json { + + implicit val jsonArrayPut: Put[List[CirceJson]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("json[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("json[]") + o.setValue(circeJsonListToPGJsonArrayString(a)) + o + } + } + + implicit val jsonArrayGet: Get[List[CirceJson]] = { + Get.Advanced + .other[PgArray]( + NonEmptyList.of("json[]") + ) + .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) + } + + } + + object Jsonb { + + implicit val jsonbArrayPut: Put[List[CirceJson]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("jsonb[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("jsonb[]") + o.setValue(circeJsonListToPGJsonArrayString(a)) + o + } + } + + } + +} diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala new file mode 100644 index 00000000..1ff37542 --- /dev/null +++ b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala @@ -0,0 +1,75 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.db.fadb.doobie + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import doobie.implicits.toSqlInterpolator +import io.circe.Json +import io.circe.syntax.EncoderOps +import org.scalatest.funsuite.AnyFunSuite +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieFunction.{DoobieMultipleResultFunction, DoobieSingleResultFunction} +import za.co.absa.db.fadb.testing.classes.DoobieTest + +import za.co.absa.db.fadb.doobie.postgres.implicits.Json.jsonArrayGet + +class JsonArrayTests extends AnyFunSuite with DoobieTest { + + class InsertActorsJson(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieSingleResultFunction[List[Actor], Unit, IO] ( + values => { + val actorsAsJsonList = values.map(_.asJson) + Seq( + { + import za.co.absa.db.fadb.doobie.postgres.implicits.Json.jsonArrayPut + fr"$actorsAsJsonList" + }, + { + import za.co.absa.db.fadb.doobie.postgres.implicits.Jsonb.jsonbArrayPut + fr"$actorsAsJsonList" + } + ) + } + ) + + class RetrieveActorsJson(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieMultipleResultFunction[Int, List[Json], IO] ( + values => Seq(fr"$values") + ) + + class RetrieveActorsJsonb(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieMultipleResultFunction[Int, List[Json], IO] ( + values => Seq(fr"$values") + ) + + private val insertActorsJson = new InsertActorsJson()(Integration, new DoobieEngine(transactor)) + + test("Retrieve Actors from json[] and jsonb[] columns"){ + val expectedActors = List(Actor(1, "John", "Doe"), Actor(2, "Jane", "Doe")) + insertActorsJson(expectedActors).unsafeRunSync() + + val retrieveActorsJson = new RetrieveActorsJson()(Integration, new DoobieEngine(transactor)) + val actualActorsJson = retrieveActorsJson(2).unsafeRunSync() + assert(expectedActors == actualActorsJson.head.map(_.as[Actor]).map(_.toTry.get)) + + val retrieveActorsJsonb = new RetrieveActorsJsonb()(Integration, new DoobieEngine(transactor)) + val actualActorsJsonb = retrieveActorsJsonb(2).unsafeRunSync() + assert(expectedActors == actualActorsJsonb.head.map(_.as[Actor]).map(_.toTry.get)) + } + +} diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala b/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala index e6cbf5ef..fa570b53 100644 --- a/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala +++ b/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala @@ -20,9 +20,15 @@ import cats.effect.IO import doobie.util.transactor.Transactor import doobie.util.transactor.Transactor.Aux import za.co.absa.db.fadb.DBSchema +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto._ trait DoobieTest { case class Actor(actorId: Int, firstName: String, lastName: String) + object Actor { + implicit val actorEncoder: Encoder[Actor] = deriveEncoder + implicit val actorDecoder: Decoder[Actor] = deriveDecoder + } case class GetActorsQueryParameters(firstName: Option[String], lastName: Option[String]) case class GetActorsByLastnameQueryParameters(lastName: String, firstName: Option[String] = None) case class CreateActorRequestBody(firstName: String, lastName: String) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 256fc732..103a8f71 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -import sbt._ +import sbt.* object Dependencies { @@ -44,7 +44,7 @@ object Dependencies { } def doobieDependencies(scalaVersion: String): Seq[ModuleID] = { - commonDependencies(scalaVersion) ++ Seq( + commonDependencies(scalaVersion) ++ jsonSerdeDependencies ++ Seq( "org.tpolecat" %% "doobie-core" % "1.0.0-RC2", "org.tpolecat" %% "doobie-hikari" % "1.0.0-RC2", "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2" @@ -56,4 +56,16 @@ object Dependencies { Seq(postgresql) } + + private def jsonSerdeDependencies: Seq[ModuleID] = { + lazy val circeCore = "io.circe" %% "circe-core" % "0.14.7" + lazy val circeParser = "io.circe" %% "circe-parser" % "0.14.7" + lazy val circeGeneric = "io.circe" %% "circe-generic" % "0.14.7" + + Seq( + circeCore, + circeParser, + circeGeneric, + ) + } } From f16a2ff7cb51f3be1c33469668f17c953db29d36 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 15:17:12 +0200 Subject: [PATCH 02/12] filename change --- .../{JsonArrayTests.scala => JsonArrayIntegrationTests.scala} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename doobie/src/test/scala/za/co/absa/db/fadb/doobie/{JsonArrayTests.scala => JsonArrayIntegrationTests.scala} (97%) diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala similarity index 97% rename from doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala rename to doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala index 1ff37542..dfcf3433 100644 --- a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala +++ b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala @@ -28,7 +28,7 @@ import za.co.absa.db.fadb.testing.classes.DoobieTest import za.co.absa.db.fadb.doobie.postgres.implicits.Json.jsonArrayGet -class JsonArrayTests extends AnyFunSuite with DoobieTest { +class JsonArrayIntegrationTests extends AnyFunSuite with DoobieTest { class InsertActorsJson(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) extends DoobieSingleResultFunction[List[Actor], Unit, IO] ( From 487d18c31c10bcd63bfe3efa6e7a27467f56c0b9 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 15:24:36 +0200 Subject: [PATCH 03/12] license --- .../fadb/doobie/postgres/implicits/package.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala index 625d1d02..cd4ef092 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.db.fadb.doobie.postgres import cats.Show From bfb5d700df11f51becebc66e6748a73c4205c370 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 21:14:11 +0200 Subject: [PATCH 04/12] refactoring --- .../{ => circe}/implicits/package.scala | 107 ++++++++---------- .../doobie/JsonArrayIntegrationTests.scala | 6 +- 2 files changed, 50 insertions(+), 63 deletions(-) rename doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/{ => circe}/implicits/package.scala (51%) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala similarity index 51% rename from doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala rename to doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index cd4ef092..2c959daa 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.db.fadb.doobie.postgres +package za.co.absa.db.fadb.doobie.postgres.circe import cats.Show import cats.data.NonEmptyList @@ -25,7 +25,7 @@ import org.postgresql.jdbc.PgArray import org.postgresql.util.PGobject import io.circe.parser._ -import scala.util.Try +import scala.util.{Failure, Success, Try} package object implicits { @@ -34,6 +34,40 @@ package object implicits { implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] .tmap(map => map.map { case (k, v) => k -> Option(v) }) + implicit val jsonArrayPut: Put[List[CirceJson]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("json[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("json[]") + o.setValue(circeJsonListToPGJsonArrayString(a)) + o + } + } + + implicit val jsonArrayGet: Get[List[CirceJson]] = { + Get.Advanced + .other[PgArray]( + NonEmptyList.of("json[]") + ) + .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) + } + + implicit val jsonbArrayPut: Put[List[CirceJson]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("jsonb[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("jsonb[]") + o.setValue(circeJsonListToPGJsonArrayString(a)) + o + } + } + private def circeJsonListToPGJsonArrayString(jsonList: List[CirceJson]): String = { val arrayElements = jsonList.map { x => // Convert to compact JSON string and escape inner quotes @@ -46,67 +80,20 @@ package object implicits { } private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[CirceJson]] = { - Try { - Option(pgArray.getArray) match { - case Some(array: Array[_]) => array.collect { - case str: String => parse(str).toTry.get - case other => parse(other.toString).toTry.get - }.toList - case None => List.empty[CirceJson] - case _ => throw new IllegalArgumentException("Unexpected type encountered.") - } - } - .toEither - .left.map(_.getMessage) - } - - object Sequence { - - implicit val get: Get[Seq[String]] = Get[List[String]].map(_.toSeq) - implicit val put: Put[Seq[String]] = Put[List[String]].contramap(_.toList) - - } - - object Json { - - implicit val jsonArrayPut: Put[List[CirceJson]] = { - Put.Advanced - .other[PGobject]( - NonEmptyList.of("json[]") - ) - .tcontramap { a => - val o = new PGobject - o.setType("json[]") - o.setValue(circeJsonListToPGJsonArrayString(a)) - o + Try(Option(pgArray.getArray)) match { + case Success(Some(array: Array[_])) => + val results = array.toList.map { + case str: String => parse(str).left.map(_.getMessage) + case other => parse(other.toString).left.map(_.getMessage) } - } - - implicit val jsonArrayGet: Get[List[CirceJson]] = { - Get.Advanced - .other[PgArray]( - NonEmptyList.of("json[]") - ) - .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) - } - - } - - object Jsonb { - - implicit val jsonbArrayPut: Put[List[CirceJson]] = { - Put.Advanced - .other[PGobject]( - NonEmptyList.of("jsonb[]") - ) - .tcontramap { a => - val o = new PGobject - o.setType("jsonb[]") - o.setValue(circeJsonListToPGJsonArrayString(a)) - o + results.partition(_.isLeft) match { + case (Nil, rights) => Right(rights.collect { case Right(json) => json }) + case (lefts, _) => Left("Failed to parse JSON: " + lefts.collect { case Left(err) => err }.mkString(", ")) } + case Success(Some(_)) => Left("Unexpected type encountered. Expected an Array.") + case Success(None) => Right(Nil) + case Failure(exception) => Left(exception.getMessage) } - } } diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala index dfcf3433..4e79dd96 100644 --- a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala +++ b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala @@ -26,7 +26,7 @@ import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieFunction.{DoobieMultipleResultFunction, DoobieSingleResultFunction} import za.co.absa.db.fadb.testing.classes.DoobieTest -import za.co.absa.db.fadb.doobie.postgres.implicits.Json.jsonArrayGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonArrayGet class JsonArrayIntegrationTests extends AnyFunSuite with DoobieTest { @@ -36,11 +36,11 @@ class JsonArrayIntegrationTests extends AnyFunSuite with DoobieTest { val actorsAsJsonList = values.map(_.asJson) Seq( { - import za.co.absa.db.fadb.doobie.postgres.implicits.Json.jsonArrayPut + import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonArrayPut fr"$actorsAsJsonList" }, { - import za.co.absa.db.fadb.doobie.postgres.implicits.Jsonb.jsonbArrayPut + import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut fr"$actorsAsJsonList" } ) From e8d83721e109c654e84cec81f698dd0d84d78909 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 21:17:24 +0200 Subject: [PATCH 05/12] refactoring --- .../doobie/postgres/circe/implicits/package.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index 2c959daa..bbedfd47 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -20,7 +20,7 @@ import cats.Show import cats.data.NonEmptyList import doobie.postgres.implicits._ import doobie.{Get, Put} -import io.circe.{Json => CirceJson} +import io.circe.Json import org.postgresql.jdbc.PgArray import org.postgresql.util.PGobject import io.circe.parser._ @@ -34,7 +34,7 @@ package object implicits { implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] .tmap(map => map.map { case (k, v) => k -> Option(v) }) - implicit val jsonArrayPut: Put[List[CirceJson]] = { + implicit val jsonArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("json[]") @@ -47,7 +47,7 @@ package object implicits { } } - implicit val jsonArrayGet: Get[List[CirceJson]] = { + implicit val jsonArrayGet: Get[List[Json]] = { Get.Advanced .other[PgArray]( NonEmptyList.of("json[]") @@ -55,7 +55,7 @@ package object implicits { .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) } - implicit val jsonbArrayPut: Put[List[CirceJson]] = { + implicit val jsonbArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("jsonb[]") @@ -68,7 +68,7 @@ package object implicits { } } - private def circeJsonListToPGJsonArrayString(jsonList: List[CirceJson]): String = { + private def circeJsonListToPGJsonArrayString(jsonList: List[Json]): String = { val arrayElements = jsonList.map { x => // Convert to compact JSON string and escape inner quotes val escapedJsonString = x.noSpaces.replace("\"", "\\\"") @@ -79,7 +79,7 @@ package object implicits { arrayElements.mkString("{", ",", "}") } - private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[CirceJson]] = { + private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[Json]] = { Try(Option(pgArray.getArray)) match { case Success(Some(array: Array[_])) => val results = array.toList.map { From f0cb4a62f25780c1ef82894bc8e9cce1e6562aae Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 21:36:12 +0200 Subject: [PATCH 06/12] refactoring --- .../db/fadb/doobie/postgres/circe/implicits/package.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index bbedfd47..0d952240 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -18,7 +18,6 @@ package za.co.absa.db.fadb.doobie.postgres.circe import cats.Show import cats.data.NonEmptyList -import doobie.postgres.implicits._ import doobie.{Get, Put} import io.circe.Json import org.postgresql.jdbc.PgArray @@ -31,9 +30,6 @@ package object implicits { private implicit val showPgArray: Show[PgArray] = Show.fromToString - implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] - .tmap(map => map.map { case (k, v) => k -> Option(v) }) - implicit val jsonArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( From 5493ab51421f80b239e714c10a19f50c91129a8f Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 21:37:45 +0200 Subject: [PATCH 07/12] refactoring --- .../fadb/doobie/postgres/circe/implicits/package.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index 0d952240..b74fe594 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -38,7 +38,7 @@ package object implicits { .tcontramap { a => val o = new PGobject o.setType("json[]") - o.setValue(circeJsonListToPGJsonArrayString(a)) + o.setValue(jsonListToPGJsonArrayString(a)) o } } @@ -48,7 +48,7 @@ package object implicits { .other[PgArray]( NonEmptyList.of("json[]") ) - .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) + .temap(pgArray => pgArrayToListOfJson(pgArray)) } implicit val jsonbArrayPut: Put[List[Json]] = { @@ -59,12 +59,12 @@ package object implicits { .tcontramap { a => val o = new PGobject o.setType("jsonb[]") - o.setValue(circeJsonListToPGJsonArrayString(a)) + o.setValue(jsonListToPGJsonArrayString(a)) o } } - private def circeJsonListToPGJsonArrayString(jsonList: List[Json]): String = { + private def jsonListToPGJsonArrayString(jsonList: List[Json]): String = { val arrayElements = jsonList.map { x => // Convert to compact JSON string and escape inner quotes val escapedJsonString = x.noSpaces.replace("\"", "\\\"") @@ -75,7 +75,7 @@ package object implicits { arrayElements.mkString("{", ",", "}") } - private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[Json]] = { + private def pgArrayToListOfJson(pgArray: PgArray): Either[String, List[Json]] = { Try(Option(pgArray.getArray)) match { case Success(Some(array: Array[_])) => val results = array.toList.map { From e425aff3b01176272935bb2b8f1b8df2be864052 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 10 Jul 2024 08:18:44 +0200 Subject: [PATCH 08/12] fix titles for jacoco reports --- .github/workflows/jacoco_report.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index bc629100..7830b727 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -69,7 +69,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{ env.coverage-overall }} min-coverage-changed-files: ${{ env.coverage-changed-files }} - title: JaCoCo `model` module code coverage report - scala ${{ env.scalaLong }} + title: JaCoCo `core` module code coverage report - scala ${{ env.scalaLong }} update-comment: true - name: Add coverage to PR (doobie) if: steps.jacocorun.outcome == 'success' @@ -80,7 +80,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{ env.coverage-overall }} min-coverage-changed-files: ${{ env.coverage-changed-files }} - title: JaCoCo `agent` module code coverage report - scala ${{ env.scalaLong }} + title: JaCoCo `doobie` module code coverage report - scala ${{ env.scalaLong }} update-comment: true - name: Add coverage to PR (slick) if: steps.jacocorun.outcome == 'success' From f7b6bdd2338c2cf9561d1beb445c41aa4e939fdd Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 11 Jul 2024 11:04:29 +0200 Subject: [PATCH 09/12] jsonOrJsonbArrayGet --- .../postgres/circe/implicits/package.scala | 16 ++++++++-------- .../fadb/doobie/JsonArrayIntegrationTests.scala | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index b74fe594..eb04d61c 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -43,14 +43,6 @@ package object implicits { } } - implicit val jsonArrayGet: Get[List[Json]] = { - Get.Advanced - .other[PgArray]( - NonEmptyList.of("json[]") - ) - .temap(pgArray => pgArrayToListOfJson(pgArray)) - } - implicit val jsonbArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( @@ -64,6 +56,14 @@ package object implicits { } } + implicit val jsonOrJsonbArrayGet: Get[List[Json]] = { + Get.Advanced + .other[PgArray]( + NonEmptyList.of("json[]") + ) + .temap(pgArray => pgArrayToListOfJson(pgArray)) + } + private def jsonListToPGJsonArrayString(jsonList: List[Json]): String = { val arrayElements = jsonList.map { x => // Convert to compact JSON string and escape inner quotes diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala index 4e79dd96..237a92dc 100644 --- a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala +++ b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala @@ -26,7 +26,7 @@ import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieFunction.{DoobieMultipleResultFunction, DoobieSingleResultFunction} import za.co.absa.db.fadb.testing.classes.DoobieTest -import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonArrayGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonOrJsonbArrayGet class JsonArrayIntegrationTests extends AnyFunSuite with DoobieTest { From b02ef125e09f6cf2fefd0c2b3683914743f98125 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 11 Jul 2024 11:06:35 +0200 Subject: [PATCH 10/12] jsonOrJsonbArrayGet --- .../absa/db/fadb/doobie/postgres/circe/implicits/package.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index eb04d61c..d1305228 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -56,6 +56,8 @@ package object implicits { } } + // to be used for both json[] and jsonb[] as it handles well both and we want to avoid collision + // when resolving implicits implicit val jsonOrJsonbArrayGet: Get[List[Json]] = { Get.Advanced .other[PgArray]( From 4f516ab313cede907f3bcdc38d08c68a352f524d Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 11 Jul 2024 12:20:49 +0200 Subject: [PATCH 11/12] make default implicits available via the same package --- .../postgres/circe/implicits/package.scala | 6 ++++++ project/Dependencies.scala | 16 +++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index d1305228..4cb526bc 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -30,6 +30,12 @@ package object implicits { private implicit val showPgArray: Show[PgArray] = Show.fromToString + implicit val jsonPut: Put[Json] = doobie.postgres.circe.json.implicits.jsonPut + implicit val jsonbPut: Put[Json] = doobie.postgres.circe.jsonb.implicits.jsonbPut + + implicit val jsonGet: Get[Json] = doobie.postgres.circe.json.implicits.jsonGet + implicit val jsonbGet: Get[Json] = doobie.postgres.circe.jsonb.implicits.jsonbGet + implicit val jsonArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 103a8f71..d3b502f0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -44,10 +44,11 @@ object Dependencies { } def doobieDependencies(scalaVersion: String): Seq[ModuleID] = { - commonDependencies(scalaVersion) ++ jsonSerdeDependencies ++ Seq( + commonDependencies(scalaVersion) ++ Seq( "org.tpolecat" %% "doobie-core" % "1.0.0-RC2", "org.tpolecat" %% "doobie-hikari" % "1.0.0-RC2", - "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2" + "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2", + "org.tpolecat" %% "doobie-postgres-circe" % "1.0.0-RC2" ) } @@ -57,15 +58,4 @@ object Dependencies { Seq(postgresql) } - private def jsonSerdeDependencies: Seq[ModuleID] = { - lazy val circeCore = "io.circe" %% "circe-core" % "0.14.7" - lazy val circeParser = "io.circe" %% "circe-parser" % "0.14.7" - lazy val circeGeneric = "io.circe" %% "circe-generic" % "0.14.7" - - Seq( - circeCore, - circeParser, - circeGeneric, - ) - } } From d255f6e1fc5ec8c463668649fb745a945001f980 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 11 Jul 2024 12:53:24 +0200 Subject: [PATCH 12/12] minors --- .../db/fadb/doobie/postgres/circe/implicits/package.scala | 4 ++-- .../co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala | 5 +++++ .../za/co/absa/db/fadb/testing/classes/DoobieTest.scala | 6 ------ project/Dependencies.scala | 3 ++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala index 4cb526bc..baf7ddaf 100644 --- a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -62,8 +62,8 @@ package object implicits { } } - // to be used for both json[] and jsonb[] as it handles well both and we want to avoid collision - // when resolving implicits + // to be used for both json[] and jsonb[] as it handles well both + // and we want to avoid collision when resolving implicits implicit val jsonOrJsonbArrayGet: Get[List[Json]] = { Get.Advanced .other[PgArray]( diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala index 237a92dc..f65e3b42 100644 --- a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala +++ b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala @@ -25,6 +25,7 @@ import org.scalatest.funsuite.AnyFunSuite import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieFunction.{DoobieMultipleResultFunction, DoobieSingleResultFunction} import za.co.absa.db.fadb.testing.classes.DoobieTest +import io.circe.generic.auto._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonOrJsonbArrayGet @@ -36,10 +37,14 @@ class JsonArrayIntegrationTests extends AnyFunSuite with DoobieTest { val actorsAsJsonList = values.map(_.asJson) Seq( { + // has to be imported inside separate scope to avoid conflicts with the import below + // as both implicits are of the same type and this would cause ambiguity import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonArrayPut fr"$actorsAsJsonList" }, { + // has to be imported inside separate scope to avoid conflicts with the import above + // as both implicits are of the same type and this would cause ambiguity import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut fr"$actorsAsJsonList" } diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala b/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala index fa570b53..e6cbf5ef 100644 --- a/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala +++ b/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala @@ -20,15 +20,9 @@ import cats.effect.IO import doobie.util.transactor.Transactor import doobie.util.transactor.Transactor.Aux import za.co.absa.db.fadb.DBSchema -import io.circe.{Decoder, Encoder} -import io.circe.generic.semiauto._ trait DoobieTest { case class Actor(actorId: Int, firstName: String, lastName: String) - object Actor { - implicit val actorEncoder: Encoder[Actor] = deriveEncoder - implicit val actorDecoder: Decoder[Actor] = deriveDecoder - } case class GetActorsQueryParameters(firstName: Option[String], lastName: Option[String]) case class GetActorsByLastnameQueryParameters(lastName: String, firstName: Option[String] = None) case class CreateActorRequestBody(firstName: String, lastName: String) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d3b502f0..1ad183b8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -48,7 +48,8 @@ object Dependencies { "org.tpolecat" %% "doobie-core" % "1.0.0-RC2", "org.tpolecat" %% "doobie-hikari" % "1.0.0-RC2", "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2", - "org.tpolecat" %% "doobie-postgres-circe" % "1.0.0-RC2" + "org.tpolecat" %% "doobie-postgres-circe" % "1.0.0-RC2", + "io.circe" %% "circe-generic" % "0.14.9" % Test ) }