Skip to content

Commit

Permalink
More updates to get bioscfg ready
Browse files Browse the repository at this point in the history
Update config, making it uniform with other controllers
Remove "store" interface
Add BMC interface for later use
Update rivets
  • Loading branch information
jakeschuurmans committed Sep 24, 2024
1 parent 562013d commit 66ac74b
Show file tree
Hide file tree
Showing 15 changed files with 730 additions and 356 deletions.
29 changes: 15 additions & 14 deletions cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,40 +9,41 @@ import (
"syscall"

"github.com/equinix-labs/otel-init-go/otelinit"
"github.com/metal-toolbox/bioscfg/internal/configuration"
"github.com/metal-toolbox/bioscfg/internal/config"
"github.com/metal-toolbox/bioscfg/internal/handlers"
"github.com/metal-toolbox/bioscfg/internal/log"
"github.com/metal-toolbox/bioscfg/internal/metrics"
"github.com/metal-toolbox/bioscfg/internal/model"
"github.com/metal-toolbox/bioscfg/internal/profiling"
"github.com/metal-toolbox/bioscfg/internal/store"
"github.com/metal-toolbox/bioscfg/internal/store/fleetdb"
"github.com/metal-toolbox/bioscfg/internal/version"
"github.com/metal-toolbox/ctrl"
)

func runWorker(ctx context.Context, args *model.Args) error {
config, err := configuration.Load(args)
cfg, err := config.Load(args.ConfigFile, args.LogLevel)
if err != nil {
slog.Error("Failed to load configuration", "error", err)
return err
}

slog.Info("Configuration loaded", config.AsLogFields()...)
slog.Info("Configuration loaded", cfg.AsLogFields()...)

log.SetLevel(config.LogLevel)
log.SetLevel(cfg.LogLevel)

// serve metrics endpoint
metrics.ListenAndServe()
version.ExportBuildInfoMetric()

if config.EnableProfiling {
if args.EnableProfiling {
profiling.Enable()
}

ctx, otelShutdown := otelinit.InitOpenTelemetry(ctx, model.AppName)
defer otelShutdown(ctx)

repository, err := store.NewRepository(ctx, config)
log.NewLogrusLogger(cfg.LogLevel)
repository, err := fleetdb.New(ctx, &cfg.Endpoints.FleetDB, log.NewLogrusLogger(cfg.LogLevel))
if err != nil {
slog.Error("Failed to create repository", "error", err)
return err
Expand All @@ -62,15 +63,15 @@ func runWorker(ctx context.Context, args *model.Args) error {

nc := ctrl.NewNatsController(
model.AppName,
config.FacilityCode,
cfg.FacilityCode,
model.AppSubject,
config.NatsConfig.NatsURL,
config.NatsConfig.CredsFile,
cfg.Endpoints.Nats.URL,
cfg.Endpoints.Nats.CredsFile,
model.AppSubject,
ctrl.WithConcurrency(config.Concurrency),
ctrl.WithKVReplicas(config.NatsConfig.KVReplicas),
ctrl.WithConnectionTimeout(config.NatsConfig.ConnectTimeout),
ctrl.WithLogger(log.NewLogrusLogger(config.LogLevel)),
ctrl.WithConcurrency(cfg.Concurrency),
ctrl.WithKVReplicas(cfg.Endpoints.Nats.KVReplicationFactor),
ctrl.WithConnectionTimeout(cfg.Endpoints.Nats.ConnectTimeout),
ctrl.WithLogger(log.NewLogrusLogger(cfg.LogLevel)),
)

if err = nc.Connect(ctx); err != nil {
Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@ go 1.22.1
require (
github.com/banzaicloud/logrus-runtime-formatter v0.0.0-20190729070250-5ae5475bae5e
github.com/bmc-toolbox/bmclib/v2 v2.2.4
github.com/bombsimon/logrusr/v2 v2.0.1
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/equinix-labs/otel-init-go v0.0.9
github.com/google/uuid v1.6.0
github.com/hashicorp/go-retryablehttp v0.7.7
github.com/jeremywohl/flatten v1.0.1
github.com/metal-toolbox/ctrl v0.2.9
github.com/metal-toolbox/fleetdb v1.19.5
github.com/metal-toolbox/rivets v1.3.8-0.20240923144748-4fa59d630b50
github.com/metal-toolbox/rivets v1.3.8
github.com/mitchellh/copystructure v1.2.0
github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/errors v0.9.1
Expand All @@ -24,6 +25,7 @@ require (
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/trace v1.28.0
golang.org/x/net v0.28.0
golang.org/x/oauth2 v0.22.0
)

Expand Down Expand Up @@ -133,7 +135,6 @@ require (
golang.org/x/arch v0.9.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v1.0.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
Expand Down Expand Up @@ -500,6 +501,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
Expand Down Expand Up @@ -551,8 +553,8 @@ github.com/metal-toolbox/ctrl v0.2.9 h1:Q1Hqpqyb71/gg2PcX/qrfoDE8FlydJt4rPQb7/Z8
github.com/metal-toolbox/ctrl v0.2.9/go.mod h1:QVATUIWFx3dbjOoEX0EnJHtRvypRlXZ9HUGaPLRyTG8=
github.com/metal-toolbox/fleetdb v1.19.5 h1:ERgdFAUtWnT/AeVhCGclsENmwPhU88JUcgOZAdxWKYI=
github.com/metal-toolbox/fleetdb v1.19.5/go.mod h1:k9MZXQsJX4NfBoANst6g1468papSs0tzsSyzN3gGWuQ=
github.com/metal-toolbox/rivets v1.3.8-0.20240923144748-4fa59d630b50 h1:v5aGsD3WnCOD6IB8o9F4XqR9kB/Vr/+LUQTmaG+aQYI=
github.com/metal-toolbox/rivets v1.3.8-0.20240923144748-4fa59d630b50/go.mod h1:8irU6eXgOa3QkjdcGi/aY4vqoMqCkbwVz7iVTYYPCX8=
github.com/metal-toolbox/rivets v1.3.8 h1:BxzBPBYPMGBwJurIe+8Xji2YL7vHZUHbOmMpszWfPYw=
github.com/metal-toolbox/rivets v1.3.8/go.mod h1:8irU6eXgOa3QkjdcGi/aY4vqoMqCkbwVz7iVTYYPCX8=
github.com/microsoft/go-mssqldb v0.17.0/go.mod h1:OkoNGhGEs8EZqchVTtochlXruEhEOaO4S0d2sB5aeGQ=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
Expand Down Expand Up @@ -671,6 +673,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/slack-go/slack v0.13.1 h1:6UkM3U1OnbhPsYeb1IMkQ6HSNOSikWluwOncJt4Tz/o=
Expand Down Expand Up @@ -1025,6 +1028,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210608053332-aa57babbf139/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
149 changes: 149 additions & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package config

import (
"os"
"strings"

"github.com/jeremywohl/flatten"
"github.com/metal-toolbox/bioscfg/internal/store/fleetdb"
"github.com/metal-toolbox/rivets/events"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/spf13/viper"
)

var (
ErrConfig = errors.New("configuration error")
)

type Configuration struct {
FacilityCode string `mapstructure:"facility"`
LogLevel string `mapstructure:"log_level"`
Endpoints Endpoints `mapstructure:"endpoints"`
Dryrun bool `mapstructure:"dryrun"`
Concurrency int `mapstructure:"concurrency"`
}

type Endpoints struct {
// NatsOptions defines the NATs events broker configuration parameters.
Nats events.NatsOptions `mapstructure:"nats"`

// FleetDBConfig defines the fleetdb client configuration parameters
FleetDB fleetdb.Config `mapstructure:"fleetdb"`
}

func Load(cfgFilePath, loglevel string) (*Configuration, error) {
v := viper.New()
cfg := &Configuration{}

err := cfg.envBindVars(v)
if err != nil {
return nil, err
}

v.SetConfigType("yaml")
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.AutomaticEnv()

err = readInFile(v, cfg, cfgFilePath)
if err != nil {
return nil, err
}

if loglevel != "" {
cfg.LogLevel = loglevel
}

err = cfg.validate()
return cfg, err
}

// Reads in the cfgFile when available and overrides from environment variables.
func readInFile(v *viper.Viper, cfg *Configuration, path string) error {
if cfg == nil {
return ErrConfig
}

if path != "" {
fh, err := os.Open(path)
if err != nil {
return errors.Wrap(ErrConfig, err.Error())
}

if err = v.ReadConfig(fh); err != nil {
return errors.Wrap(ErrConfig, "ReadConfig error:"+err.Error())
}
} else {
v.AddConfigPath(".")
v.SetConfigName("config")
err := v.ReadInConfig()
if err != nil {
return err
}
}

err := v.Unmarshal(cfg)
if err != nil {
return err
}

return nil
}

func (cfg *Configuration) validate() error {
if cfg == nil {
return ErrConfig
}

if cfg.FacilityCode == "" {
return errors.Wrap(ErrConfig, "no facility codes")
}

if cfg.LogLevel == "" {
cfg.LogLevel = "info"
}

if cfg.Concurrency == 0 {
cfg.Concurrency = 1
}

return nil
}

// envBindVars binds environment variables to the struct
// without a configuration file being unmarshalled,
// this is a workaround for a viper bug,
//
// This can be replaced by the solution in https://github.com/spf13/viper/pull/1429
// once that PR is merged.
func (cfg *Configuration) envBindVars(v *viper.Viper) error {
envKeysMap := map[string]interface{}{}
if err := mapstructure.Decode(cfg, &envKeysMap); err != nil {
return err
}

// Flatten nested conf map
flat, err := flatten.Flatten(envKeysMap, "", flatten.DotStyle)
if err != nil {
return errors.Wrap(err, "Unable to flatten config")
}

for k := range flat {
if err := v.BindEnv(k); err != nil {
return errors.Wrap(ErrConfig, "env var bind error: "+err.Error())
}
}

return nil
}

func (cfg *Configuration) AsLogFields() []any {
return []any{
"logLevel", cfg.LogLevel,
"concurrency", cfg.Concurrency,
"facilityCode", cfg.FacilityCode,
"authenticate", cfg.Endpoints.FleetDB.Authenticate,
"fleetDBUrl", cfg.Endpoints.FleetDB.URL,
"natsURL", cfg.Endpoints.Nats.URL,
}
}
Loading

0 comments on commit 66ac74b

Please sign in to comment.