-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #253 from pluralsight/feature/bootstrapLogic
Bootstrap Topic Creation
- Loading branch information
Showing
9 changed files
with
221 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
package hydra.ingest.app | ||
|
||
import cats.implicits._ | ||
import ciris.{ConfigValue, _} | ||
import hydra.kafka.config.MetadataSchemaConfig | ||
import hydra.kafka.model.Subject | ||
import org.apache.avro.Schema | ||
|
||
import scala.concurrent.duration._ | ||
|
||
object AppConfig { | ||
|
||
final case class SchemaRegistryConfig( | ||
fullUrl: String, | ||
maxCacheSize: Int | ||
) | ||
|
||
private val schemaRegistryConfig: ConfigValue[SchemaRegistryConfig] = | ||
( | ||
env("HYDRA_SCHEMA_REGISTRY_URL").as[String].default("http://localhost:8081"), | ||
env("HYDRA_MAX_SCHEMAS_PER_SUBJECT").as[Int].default(1000) | ||
) | ||
.parMapN(SchemaRegistryConfig) | ||
|
||
final case class CreateTopicConfig( | ||
schemaRegistryConfig: SchemaRegistryConfig, | ||
numRetries: Int, | ||
baseBackoffDelay: FiniteDuration | ||
) | ||
|
||
private val createTopicConfig: ConfigValue[CreateTopicConfig] = | ||
( | ||
schemaRegistryConfig, | ||
env("CREATE_TOPIC_NUM_RETRIES").as[Int].default(1), | ||
env("CREATE_TOPIC_BASE_BACKOFF_DELAY").as[FiniteDuration].default(1.second) | ||
) | ||
.parMapN(CreateTopicConfig) | ||
|
||
private implicit val subjectConfigDecoder: ConfigDecoder[String, Subject] = | ||
ConfigDecoder[String, String].mapOption("Subject")(Subject.createValidated) | ||
|
||
final case class V2MetadataTopicConfig( | ||
topicName: Subject, | ||
keySchema: Schema, | ||
valueSchema: Schema, | ||
createOnStartup: Boolean | ||
) | ||
|
||
private val v2MetadataTopicConfig: ConfigValue[V2MetadataTopicConfig] = | ||
( | ||
env("HYDRA_V2_METADATA_TOPIC_NAME").as[Subject].default(Subject.createValidated("_hydra.v2.metadata").get), | ||
env("HYDRA_V2_METADATA_CREATE_ON_STARTUP").as[Boolean].default(false) | ||
) | ||
.parMapN { (subject, createOnStartup) => | ||
V2MetadataTopicConfig(subject, MetadataSchemaConfig.keySchema, MetadataSchemaConfig.valueSchema, createOnStartup) | ||
} | ||
|
||
final case class AppConfig( | ||
createTopicConfig: CreateTopicConfig, | ||
v2MetadataTopicConfig: V2MetadataTopicConfig | ||
) | ||
|
||
val appConfig: ConfigValue[AppConfig] = | ||
( | ||
createTopicConfig, | ||
v2MetadataTopicConfig | ||
) | ||
.parMapN(AppConfig) | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
package hydra.ingest.modules | ||
|
||
import hydra.avro.registry.SchemaRegistry | ||
import hydra.ingest.app.AppConfig.SchemaRegistryConfig | ||
import cats.effect.Sync | ||
import cats.implicits._ | ||
|
||
final class Algebras[F[_]] private ( | ||
val schemaRegistry: SchemaRegistry[F] | ||
) | ||
|
||
object Algebras { | ||
def make[F[_]: Sync]( | ||
schemaRegistryConfig: SchemaRegistryConfig | ||
): F[Algebras[F]] = | ||
for { | ||
schemaRegistry <- SchemaRegistry.live[F](schemaRegistryConfig.fullUrl, schemaRegistryConfig.maxCacheSize) | ||
} yield new Algebras[F](schemaRegistry) | ||
} |
33 changes: 33 additions & 0 deletions
33
ingest/src/main/scala/hydra.ingest/modules/Bootstrap.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
package hydra.ingest.modules | ||
|
||
import cats.effect.Sync | ||
import hydra.core.bootstrap.CreateTopicProgram | ||
import cats.implicits._ | ||
import hydra.ingest.app.AppConfig.V2MetadataTopicConfig | ||
|
||
final class Bootstrap[F[_]: Sync] private ( | ||
createTopicProgram: CreateTopicProgram[F], | ||
cfg: V2MetadataTopicConfig | ||
) { | ||
|
||
def bootstrapAll: F[Unit] = for { | ||
_ <- bootstrapMetadataTopic | ||
} yield () | ||
|
||
private def bootstrapMetadataTopic: F[Unit] = | ||
if (cfg.createOnStartup) { | ||
createTopicProgram.createTopic(cfg.topicName.value, cfg.keySchema, cfg.valueSchema) | ||
} else { | ||
Sync[F].unit | ||
} | ||
|
||
} | ||
|
||
object Bootstrap { | ||
def make[F[_]: Sync]( | ||
createTopicProgram: CreateTopicProgram[F], | ||
v2MetadataTopicConfig: V2MetadataTopicConfig | ||
): F[Bootstrap[F]] = Sync[F].delay { | ||
new Bootstrap[F](createTopicProgram, v2MetadataTopicConfig) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
package hydra.ingest.modules | ||
|
||
import cats.effect._ | ||
import cats.implicits._ | ||
import hydra.core.bootstrap.CreateTopicProgram | ||
import hydra.ingest.app.AppConfig.CreateTopicConfig | ||
import io.chrisdavenport.log4cats.Logger | ||
import retry.RetryPolicies._ | ||
import retry.RetryPolicy | ||
|
||
final class Programs[F[_]: Logger: Sync: Timer] private ( | ||
cfg: CreateTopicConfig, | ||
algebras: Algebras[F] | ||
) { | ||
|
||
val retryPolicy: RetryPolicy[F] = | ||
limitRetries[F](cfg.numRetries) |+| exponentialBackoff[F](cfg.baseBackoffDelay) | ||
|
||
val createTopic: CreateTopicProgram[F] = new CreateTopicProgram[F]( | ||
algebras.schemaRegistry, | ||
retryPolicy | ||
) | ||
|
||
} | ||
|
||
object Programs { | ||
|
||
def make[F[_]: Logger: Sync: Timer]( | ||
createTopicConfig: CreateTopicConfig, | ||
algebras: Algebras[F] | ||
): F[Programs[F]] = Sync[F].delay { | ||
new Programs[F](createTopicConfig, algebras) | ||
} | ||
|
||
} |
19 changes: 19 additions & 0 deletions
19
kafka/src/main/scala/hydra/kafka/config/MetadataSchemaConfig.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
package hydra.kafka.config | ||
|
||
import org.apache.avro.Schema | ||
import org.apache.avro.Schema.Parser | ||
|
||
import scala.io.Source | ||
|
||
object MetadataSchemaConfig { | ||
|
||
private def parseSchema(fileName: String): Schema = { | ||
val jsonSchemaString: String = Source.fromResource(fileName).mkString | ||
new Parser().parse(jsonSchemaString) | ||
} | ||
|
||
lazy val keySchema: Schema = parseSchema("HydraMetadataTopicKeyV2.avsc") | ||
|
||
lazy val valueSchema: Schema = parseSchema("HydraMetadataTopicValueV2.avsc") | ||
|
||
} |
23 changes: 0 additions & 23 deletions
23
kafka/src/test/scala/hydra/kafka/HydraMetadataTopicV2SchemaSpec.scala
This file was deleted.
Oops, something went wrong.
16 changes: 16 additions & 0 deletions
16
kafka/src/test/scala/hydra/kafka/config/MetadataSchemaConfigSpec.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
package hydra.kafka.config | ||
|
||
import org.apache.avro.Schema | ||
import org.scalatest.{FlatSpec, Matchers} | ||
|
||
class MetadataSchemaConfigSpec extends FlatSpec with Matchers { | ||
|
||
it should "be able to parse the HydraMetadataTopicValueV2.avsc into the Schema class" in { | ||
MetadataSchemaConfig.keySchema shouldBe a[Schema] | ||
} | ||
|
||
it should "be able to parse the HydraMetadataTopicKeyV2.avsc into the Schema Class" in { | ||
MetadataSchemaConfig.valueSchema shouldBe a[Schema] | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters