Implement our own environment variable lookup

Based on https://gist.github.com/vooon/911c13af31c4aff53477536d8d6afe65

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/aws/aws-sdk-go v1.55.5 // indirect
 	github.com/bodgit/ntlmssp v0.0.0-20240506230425-31973bb52d9b // indirect
 	github.com/bodgit/windows v1.0.1 // indirect
+	github.com/caarlos0/env/v11 v11.2.2 // indirect
 	github.com/coreos/go-json v0.0.0-20231102161613-e49c8866685a // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
@@ -28,6 +29,7 @@ require (
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/gophercloud/utils/v2 v2.0.0-20241209100706-e3a3b7c07d26 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-plugin v1.6.2 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
 github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
 github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
+github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg=
+github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc=
 github.com/coreos/go-json v0.0.0-20231102161613-e49c8866685a h1:QimUZQ6Au5wFKKkPMmdoXen+CNR66lXt/76AQLBltS0=
 github.com/coreos/go-json v0.0.0-20231102161613-e49c8866685a/go.mod h1:rcFZM3uxVvdyNmsAV2jopgPD1cs5SPWJWU5dOz2LUnw=
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
@@ -37,6 +39,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/gophercloud/gophercloud/v2 v2.3.0 h1:5ipI2Mgxee0TwQxqnOIUdTbzL4ZBB8GORyZko+yGXI0=
 github.com/gophercloud/gophercloud/v2 v2.3.0/go.mod h1:uJWNpTgJPSl2gyzJqcU/pIAhFUWvIkp8eE8M15n9rs4=
+github.com/gophercloud/utils/v2 v2.0.0-20241209100706-e3a3b7c07d26 h1:N65GYmx5LrMeYdeXcxMESDU+2pDyAOXlFNlHl7siUwM=
+github.com/gophercloud/utils/v2 v2.0.0-20241209100706-e3a3b7c07d26/go.mod h1:7SHUbtoiSYINNKgAVxse+PMhIio05IK7shHy8DVRaN0=
 github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=

internal/openstackclient/openstackclient.go

@@ -2,31 +2,55 @@ package openstackclient
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
-	"os"
+	"net/http"
 
+	"github.com/caarlos0/env/v11"
 	"github.com/gophercloud/gophercloud/v2"
 	"github.com/gophercloud/gophercloud/v2/openstack"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers"
 	"github.com/gophercloud/gophercloud/v2/openstack/config"
 	"github.com/gophercloud/gophercloud/v2/openstack/config/clouds"
 	"github.com/gophercloud/gophercloud/v2/openstack/image/v2/images"
 	"github.com/gophercloud/gophercloud/v2/openstack/utils"
+	osClient "github.com/gophercloud/utils/v2/client"
 	"github.com/mitchellh/mapstructure"
 )
 
-type AuthConfig struct {
-	// AuthFromEnv specifies whether to use environment variables for auth
-	AuthFromEnv bool
+type AuthConfig interface {
+	Parse() (gophercloud.AuthOptions, gophercloud.EndpointOpts, *tls.Config, error)
+	HTTPOpts() (debug bool, computeApiVersion string)
+}
 
-	// Cloud is the name of the cloud config from clouds.yaml to use
-	Cloud string
+type CloudOpts struct {
+	AllowReauth bool `envDefault:"true"`
+}
 
-	// CloudsConfig is the path to the clouds.yaml file
-	CloudsConfig string
+type CloudConfig struct {
+	ClientConfigFile  string `json:"client-config-file" env:"OS_CLIENT_CONFIG_FILE"`
+	Cloud             string `json:"cloud" env:"OS_CLOUD"`
+	RegionName        string `json:"region-name" env:"OS_REGION_NAME"`
+	EndpointType      string `json:"endpoint-type" env:"OS_INTERFACE,OS_ENDPOINT_TYPE"`
+	Debug             bool   `json:"debug" env:"OS_DEBUG"`
+	ComputeApiVersion string `json:"compute-api-version" env:"OS_COMPUTE_API_VERSION" envDefault:"2.79"`
+}
 
-	// NovaMicroversion is the microversion of the OpenStack Nova client. Default 2.79 (which should be ok for Train+)
-	NovaMicroversion string
+type EnvCloudConfig struct {
+	CloudConfig `embed:"" yaml:",inline"`
+
+	AuthURL                     string `json:"auth-url" env:"OS_AUTH_URL"`
+	Username                    string `json:"username" env:"OS_USERNAME"`
+	UserID                      string `json:"user-id" env:"OS_USER_ID"`
+	Password                    string `json:"password" env:"OS_PASSWORD"`
+	Passcode                    string `json:"passcode" env:"OS_PASSCODE"`
+	ProjectName                 string `json:"project-name" env:"OS_PROJECT_NAME"`
+	ProjectID                   string `json:"project-id" env:"OS_PROJECT_ID"`
+	UserDomainName              string `json:"user-domain-name" env:"OS_USER_DOMAIN_NAME,OS_DOMAIN_NAME"`
+	UserDomainID                string `json:"user-domain-id" env:"OS_USER_DOMAIN_ID,OS_DOMAIN_ID"`
+	ApplicationCredentialID     string `json:"application-credential-id" env:"OS_APPLICATION_CREDENTIAL_ID"`
+	ApplicationCredentialName   string `json:"application-credential-name" env:"OS_APPLICATION_CREDENTIAL_NAME"`
+	ApplicationCredentialSecret string `json:"application-credential-secret" env:"OS_APPLICATION_CREDENTIAL_SECRET"`
 }
 
 // Some good known properties useful for setting up ConnectInfo
@@ -58,34 +82,36 @@ type Client interface {
 	DeleteServer(ctx context.Context, serverId string) error
 }
 
-var _ Client = (*client)(nil)
-
 type client struct {
 	compute *gophercloud.ServiceClient
 	image   *gophercloud.ServiceClient
 }
 
-func New(ctx context.Context, authConfig AuthConfig) (Client, error) {
-	if authConfig.NovaMicroversion == "" {
-		authConfig.NovaMicroversion = "2.79" // Train+
+func New(ctx context.Context, authConfig AuthConfig, cloudOpts *CloudOpts) (Client, error) {
+	if cloudOpts == nil {
+		cloudOpts = &CloudOpts{}
 	}
 
-	providerClient, endpointOps, err := newProviderClient(ctx, authConfig)
+	var err error
+	err = env.Parse(&cloudOpts)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to parse cloudOpts: %w", err)
 	}
 
-	computeClient, err := openstack.NewComputeV2(providerClient, endpointOps)
+	err = env.Parse(&authConfig)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to parse authConfig: %w", err)
 	}
 
-	_computeClient, err := utils.RequireMicroversion(ctx, *computeClient, authConfig.NovaMicroversion)
+	providerClient, endpointOps, err := NewProviderClient(ctx, authConfig, cloudOpts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to request microversion %s for OpenStack Nova: %w", authConfig.NovaMicroversion, err)
+		return nil, err
 	}
 
-	computeClient = &_computeClient
+	computeClient, err := NewComputeClient(ctx, providerClient, endpointOps, authConfig)
+	if err != nil {
+		return nil, err
+	}
 
 	imageClient, err := openstack.NewImageV2(providerClient, endpointOps)
 	if err != nil {
@@ -98,42 +124,120 @@ func New(ctx context.Context, authConfig AuthConfig) (Client, error) {
 	}, nil
 }
 
-func newProviderClient(ctx context.Context, authConfig AuthConfig) (*gophercloud.ProviderClient, gophercloud.EndpointOpts, error) {
-	if authConfig.AuthFromEnv {
-		var err error
-		endpointOps := gophercloud.EndpointOpts{Region: os.Getenv("OS_REGION_NAME")}
-		authOptions, err := openstack.AuthOptionsFromEnv()
+func (cloudConfig *CloudConfig) HTTPOpts() (debug bool, computeApiVersion string) {
+	return cloudConfig.Debug, cloudConfig.ComputeApiVersion
+}
+
+func (cloudConfig *CloudConfig) Parse() (gophercloud.AuthOptions, gophercloud.EndpointOpts, *tls.Config, error) {
+	parseOpts := []clouds.ParseOption{clouds.WithCloudName(cloudConfig.Cloud)}
+	if cloudConfig.ClientConfigFile != "" {
+		parseOpts = append(parseOpts, clouds.WithLocations(cloudConfig.ClientConfigFile))
+	}
+
+	authOptions, endpointOpts, tlsCfg, err := clouds.Parse(parseOpts...)
+	if err != nil {
+		return gophercloud.AuthOptions{}, gophercloud.EndpointOpts{}, nil, err
+	}
+
+	if cloudConfig.RegionName != "" {
+		endpointOpts.Region = cloudConfig.RegionName
+	}
+	if cloudConfig.EndpointType != "" {
+		endpointOpts.Availability = gophercloud.Availability(cloudConfig.EndpointType)
+	}
+
+	return authOptions, endpointOpts, tlsCfg, nil
+}
+
+func (envCloudConfig *EnvCloudConfig) Parse() (gophercloud.AuthOptions, gophercloud.EndpointOpts, *tls.Config, error) {
+	if envCloudConfig.Cloud != "" {
+		authOptions, endpointOpts, tlsCfg, err := envCloudConfig.CloudConfig.Parse()
 		if err != nil {
-			return nil, gophercloud.EndpointOpts{}, fmt.Errorf("failed to get auth options from environment: %w", err)
+			return gophercloud.AuthOptions{}, gophercloud.EndpointOpts{}, nil, err
 		}
-		authOptions.AllowReauth = true
 
-		providerClient, err := openstack.AuthenticatedClient(ctx, authOptions)
-		if err != nil {
-			return nil, gophercloud.EndpointOpts{}, fmt.Errorf("failed to connect to OpenStack Keystone: %w", err)
+		if envCloudConfig.ProjectName != "" {
+			authOptions.TenantName = envCloudConfig.ProjectName
+			authOptions.TenantID = ""
 		}
-		return providerClient, endpointOps, nil
+		if envCloudConfig.ProjectID != "" {
+			authOptions.TenantID = envCloudConfig.ProjectID
+			authOptions.TenantName = ""
+		}
+
+		return authOptions, endpointOpts, tlsCfg, nil
+	}
+
+	authOptions := gophercloud.AuthOptions{
+		IdentityEndpoint:            envCloudConfig.AuthURL,
+		UserID:                      envCloudConfig.UserID,
+		Username:                    envCloudConfig.Username,
+		Password:                    envCloudConfig.Password,
+		Passcode:                    envCloudConfig.Passcode,
+		TenantID:                    envCloudConfig.ProjectID,
+		TenantName:                  envCloudConfig.ProjectName,
+		DomainID:                    envCloudConfig.UserDomainID,
+		DomainName:                  envCloudConfig.UserDomainName,
+		ApplicationCredentialID:     envCloudConfig.ApplicationCredentialID,
+		ApplicationCredentialName:   envCloudConfig.ApplicationCredentialName,
+		ApplicationCredentialSecret: envCloudConfig.ApplicationCredentialSecret,
+	}
+
+	endpointOpts := gophercloud.EndpointOpts{
+		Region:       envCloudConfig.RegionName,
+		Availability: gophercloud.Availability(envCloudConfig.EndpointType),
 	}
 
-	cloudOpts := []clouds.ParseOption{clouds.WithCloudName(authConfig.Cloud)}
-	if authConfig.CloudsConfig != "" {
-		cloudOpts = append(cloudOpts, clouds.WithLocations(authConfig.CloudsConfig))
+	return authOptions, endpointOpts, nil, nil
+}
+
+func NewHTTPClient(tlsCfg *tls.Config) http.Client {
+	httpClient := http.Client{
+		Transport: http.DefaultTransport.(*http.Transport).Clone(),
+	}
+
+	if tlsCfg != nil {
+		tr := httpClient.Transport.(*http.Transport)
+		tr.TLSClientConfig = tlsCfg
 	}
 
-	authOptions, endpointOps, tlsCfg, err := clouds.Parse(cloudOpts...)
+	httpClient.Transport = &osClient.RoundTripper{
+		Rt: httpClient.Transport,
+	}
+	return httpClient
+}
+
+func NewProviderClient(ctx context.Context, authConfig AuthConfig, cloudOpts *CloudOpts) (*gophercloud.ProviderClient, gophercloud.EndpointOpts, error) {
+	authOptions, endpointOpts, tlsCfg, err := authConfig.Parse()
+	if err != nil {
+		return nil, gophercloud.EndpointOpts{}, err
+	}
+
+	httpClient := NewHTTPClient(tlsCfg)
+	authOptions.AllowReauth = cloudOpts.AllowReauth
+
+	providerClient, err := config.NewProviderClient(ctx, authOptions, config.WithHTTPClient(httpClient))
 	if err != nil {
-		return nil, gophercloud.EndpointOpts{}, fmt.Errorf("failed to parse clouds.yaml: %w", err)
+		return nil, gophercloud.EndpointOpts{}, err
 	}
 
-	// plugin is a long running process. force allow reauth
-	authOptions.AllowReauth = true
+	return providerClient, endpointOpts, nil
+}
+
+func NewComputeClient(ctx context.Context, providerClient *gophercloud.ProviderClient, endpointOps gophercloud.EndpointOpts, authConfig AuthConfig) (*gophercloud.ServiceClient, error) {
+	_, computeApiVersion := authConfig.HTTPOpts()
+
+	computeClient, err := openstack.NewComputeV2(providerClient, endpointOps)
+	if err != nil {
+		return &gophercloud.ServiceClient{}, err
+	}
 
-	providerClient, err := config.NewProviderClient(ctx, authOptions, config.WithTLSConfig(tlsCfg))
+	_computeClient, err := utils.RequireMicroversion(ctx, *computeClient, computeApiVersion)
 	if err != nil {
-		return nil, gophercloud.EndpointOpts{}, fmt.Errorf("failed to connect to OpenStack Keystone: %w", err)
+		return &gophercloud.ServiceClient{}, err
 	}
 
-	return providerClient, endpointOps, nil
+	return &_computeClient, err
 }
 
 func (c *client) GetImageProperties(ctx context.Context, imageRef string) (*ImageProperties, error) {

provider.go

@@ -25,7 +25,6 @@ var _ provider.InstanceGroup = (*InstanceGroup)(nil)
 type InstanceGroup struct {
 	Cloud            string        `json:"cloud"`             // cloud to use
 	CloudsConfig     string        `json:"clouds_config"`     // optional: path to clouds.yaml
-	AuthFromEnv      bool          `json:"auth_from_env"`     // optional: Use environment variables for authentication
 	Name             string        `json:"name"`              // name of the cluster
 	NovaMicroversion string        `json:"nova_microversion"` // Microversion for the Nova client
 	ServerSpec       ExtCreateOpts `json:"server_spec"`       // instance creation spec
@@ -46,12 +45,11 @@ func (g *InstanceGroup) Init(ctx context.Context, log hclog.Logger, settings pro
 	g.log.Debug("Initializing fleeting-plugin-openstack")
 
 	var err error
-	g.client, err = openstackclient.New(ctx, openstackclient.AuthConfig{
-		AuthFromEnv:      g.AuthFromEnv,
-		Cloud:            g.Cloud,
-		CloudsConfig:     g.CloudsConfig,
-		NovaMicroversion: g.NovaMicroversion,
-	})
+	g.client, err = openstackclient.New(ctx, &openstackclient.CloudConfig{
+		ClientConfigFile:  g.CloudsConfig,
+		Cloud:             g.Cloud,
+		ComputeApiVersion: g.NovaMicroversion,
+	}, nil)
 
 	if err != nil {
 		return provider.ProviderInfo{}, err