Skip to content

Commit

Permalink
feat(application): define schema for storage flag in application reso…
Browse files Browse the repository at this point in the history
…urce

This commit extends the application schema to support the storage flag in the Terraform provider for Juju.
The schema now includes varios attributes for configuring the storage constraints of a Juju application.
feat(application) update CRUD function for application resource

This commit includes several updates to the resource CRUD functions in the Juju provider. The changes include:

- Added error handling for application not found in `handleApplicationNotFoundError` function.
- Added storage conintes to Create and Read functions
- Modifier retry process to read application changes, to be able to wait
  for storage creation

test(application): add acceptance test for resource application storage

feat: introduce storage_directives map instead storage set

docs: modify placement example for resource_application to include
storage
  • Loading branch information
anvial committed Jul 2, 2024
1 parent ca1da69 commit f1a2dc9
Show file tree
Hide file tree
Showing 9 changed files with 664 additions and 91 deletions.
14 changes: 6 additions & 8 deletions docs/resources/application.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ resource "juju_application" "placement_example" {
latest revision.
* If the charm revision or channel are not updated, then no changes will take
place (juju does not have an "un-attach" command for resources).
- `storage` (Attributes Set) Configure storage constraints for the juju application. (see [below for nested schema](#nestedatt--storage))
- `storage` (Attributes Set) Storage used by the application. (see [below for nested schema](#nestedatt--storage))
- `storage_directives` (Map of String) Storage directives (constraints) for the juju application. The map key is the label of the storage defined by the charm, the map value is the storage directive in the form <pool>,<count>,<size>.
- `trust` (Boolean) Set the trust for the application.
- `units` (Number) The number of application units to deploy for the charm.

Expand Down Expand Up @@ -127,15 +128,12 @@ Optional:
<a id="nestedatt--storage"></a>
### Nested Schema for `storage`

Required:

- `label` (String) The specific storage option defined in the charm.

Optional:
Read-Only:

- `count` (Number) The number of volumes.
- `pool` (String) Name of the storage pool to use. E.g. ebs on aws.
- `size` (String) The size of each volume. E.g. 100G.
- `label` (String) The specific storage option defined in the charm.
- `pool` (String) Name of the storage pool to use.
- `size` (String) The size of each volume.

## Import

Expand Down
6 changes: 5 additions & 1 deletion examples/resources/juju_application/resource.tf
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ resource "juju_application" "this" {
}
}

resource "juju_application" "placement_example" {
resource "juju_application" "placement_and_storage_example" {
name = "placement-example"
model = juju_model.development.name
charm {
Expand All @@ -30,6 +30,10 @@ resource "juju_application" "placement_example" {
units = 3
placement = "0,1,2"

storage = {
files = "101M"
}

config = {
external-hostname = "..."
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
)

require (
github.com/dustin/go-humanize v1.0.1
github.com/hashicorp/terraform-plugin-framework v1.9.0
github.com/hashicorp/terraform-plugin-framework-validators v0.12.0
github.com/hashicorp/terraform-plugin-go v0.23.0
Expand Down Expand Up @@ -69,7 +70,6 @@ require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
Expand Down
142 changes: 119 additions & 23 deletions internal/juju/applications.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
"github.com/juju/juju/core/network"
"github.com/juju/juju/environs/config"
"github.com/juju/juju/rpc/params"
jujustorage "github.com/juju/juju/storage"
jujuversion "github.com/juju/juju/version"
"github.com/juju/names/v5"
"github.com/juju/retry"
Expand All @@ -59,6 +60,17 @@ func (ae *applicationNotFoundError) Error() string {
return fmt.Sprintf("application %s not found", ae.appName)
}

var StorageNotFoundError = &storageNotFoundError{}

// StorageNotFoundError
type storageNotFoundError struct {
storageName string
}

func (se *storageNotFoundError) Error() string {
return fmt.Sprintf("storage %s not found", se.storageName)
}

type applicationsClient struct {
SharedClient
controllerVersion version.Number
Expand Down Expand Up @@ -125,21 +137,22 @@ func ConfigEntryToString(input interface{}) string {
}

type CreateApplicationInput struct {
ApplicationName string
ModelName string
CharmName string
CharmChannel string
CharmBase string
CharmSeries string
CharmRevision int
Units int
Trust bool
Expose map[string]interface{}
Config map[string]string
Placement string
Constraints constraints.Value
EndpointBindings map[string]string
Resources map[string]int
ApplicationName string
ModelName string
CharmName string
CharmChannel string
CharmBase string
CharmSeries string
CharmRevision int
Units int
Trust bool
Expose map[string]interface{}
Config map[string]string
Placement string
Constraints constraints.Value
EndpointBindings map[string]string
Resources map[string]int
StorageConstraints map[string]jujustorage.Constraints
}

// validateAndTransform returns transformedCreateApplicationInput which
Expand All @@ -156,6 +169,7 @@ func (input CreateApplicationInput) validateAndTransform(conn api.Connection) (p
parsed.trust = input.Trust
parsed.units = input.Units
parsed.resources = input.Resources
parsed.storage = input.StorageConstraints

appName := input.ApplicationName
if appName == "" {
Expand Down Expand Up @@ -241,6 +255,7 @@ type transformedCreateApplicationInput struct {
trust bool
endpointBindings map[string]string
resources map[string]int
storage map[string]jujustorage.Constraints
}

type CreateApplicationResponse struct {
Expand All @@ -266,6 +281,7 @@ type ReadApplicationResponse struct {
Principal bool
Placement string
EndpointBindings map[string]string
Storage map[string]jujustorage.Constraints
Resources map[string]int
}

Expand All @@ -282,10 +298,11 @@ type UpdateApplicationInput struct {
Unexpose []string
Config map[string]string
//Series string // Unsupported today
Placement map[string]interface{}
Constraints *constraints.Value
EndpointBindings map[string]string
Resources map[string]int
Placement map[string]interface{}
Constraints *constraints.Value
EndpointBindings map[string]string
StorageConstraints map[string]jujustorage.Constraints
Resources map[string]int
}

type DestroyApplicationInput struct {
Expand Down Expand Up @@ -318,6 +335,9 @@ func (c applicationsClient) CreateApplication(ctx context.Context, input *Create
return nil, err
}

// print transformedInput.storage
c.Tracef("transformedInput.storage", map[string]interface{}{"transformedInput.storage": transformedInput.storage})

applicationAPIClient := apiapplication.NewClient(conn)
if applicationAPIClient.BestAPIVersion() >= 19 {
err = c.deployFromRepository(applicationAPIClient, transformedInput)
Expand Down Expand Up @@ -364,6 +384,7 @@ func (c applicationsClient) deployFromRepository(applicationAPIClient *apiapplic
Revision: &transformedInput.charmRevision,
Trust: transformedInput.trust,
Resources: resources,
Storage: transformedInput.storage,
})
return errors.Join(errs...)
}
Expand Down Expand Up @@ -734,7 +755,7 @@ func (c applicationsClient) ReadApplicationWithRetryOnNotFound(ctx context.Conte
Func: func() error {
var err error
output, err = c.ReadApplication(input)
if errors.As(err, &ApplicationNotFoundError) {
if errors.As(err, &ApplicationNotFoundError) || errors.As(err, &StorageNotFoundError) {
return err
} else if err != nil {
// Log the error to the terraform Diagnostics to be
Expand All @@ -759,6 +780,18 @@ func (c applicationsClient) ReadApplicationWithRetryOnNotFound(ctx context.Conte
if len(machines) != output.Units {
return fmt.Errorf("ReadApplicationWithRetryOnNotFound: need %d machines, have %d", output.Units, len(machines))
}

// NOTE: Application can always have storage. However, they
// will not be listed right after the application is created. So
// we need to wait for the storages to be ready. And we need to
// check if all storage constraints have pool equal "" and size equal 0
// to drop the error.
for _, storage := range output.Storage {
if storage.Pool == "" || storage.Size == 0 {
return fmt.Errorf("ReadApplicationWithRetryOnNotFound: no storages found in output")
}
}

// NOTE: An IAAS subordinate should also have machines. However, they
// will not be listed until after the relation has been created.
// Those happen with the integration resource which will not be
Expand All @@ -785,6 +818,50 @@ func (c applicationsClient) ReadApplicationWithRetryOnNotFound(ctx context.Conte
return output, retryErr
}

func transformToStorageConstraints(
storageDetailsSlice []params.StorageDetails,
filesystemDetailsSlice []params.FilesystemDetails,
volumeDetailsSlice []params.VolumeDetails,
) map[string]jujustorage.Constraints {
storage := make(map[string]jujustorage.Constraints)
for _, storageDetails := range storageDetailsSlice {
// switch base on storage kind
storageCounters := make(map[string]uint64)
switch storageDetails.Kind.String() {
case "filesystem":
for _, fd := range filesystemDetailsSlice {
if fd.Storage.StorageTag == storageDetails.StorageTag {
// Cut 'storage-' prefix from the storage tag and `-NUMBER` suffix
storageLabel := getStorageLabel(storageDetails)
storageCounters[storageLabel]++
storage[storageLabel] = jujustorage.Constraints{
Pool: fd.Info.Pool,
Size: fd.Info.Size,
Count: storageCounters[storageLabel],
}
}
}
case "block":
for _, vd := range volumeDetailsSlice {
if vd.Storage.StorageTag == storageDetails.StorageTag {
storageLabel := getStorageLabel(storageDetails)
storageCounters[storageLabel]++
storage[storageLabel] = jujustorage.Constraints{
Pool: vd.Info.Pool,
Size: vd.Info.Size,
Count: storageCounters[storageLabel],
}
}
}
}
}
return storage
}

func getStorageLabel(storageDetails params.StorageDetails) string {
return strings.TrimSuffix(strings.TrimPrefix(storageDetails.StorageTag, "storage-"), "-0")
}

func (c applicationsClient) ReadApplication(input *ReadApplicationInput) (*ReadApplicationResponse, error) {
conn, err := c.GetConnection(&input.ModelName)
if err != nil {
Expand Down Expand Up @@ -815,10 +892,20 @@ func (c applicationsClient) ReadApplication(input *ReadApplicationInput) (*ReadA

appInfo := apps[0].Result

// TODO: Investigate why we're getting the full status here when
// application status is needed.
status, err := clientAPIClient.Status(nil)
// We are currently retrieving the full status, which might be more information than necessary.
// The main focus is on the application status, particularly including the storage status.
// It might be more efficient to directly query for the application status and its associated storage status.
// TODO: Investigate if there's a way to optimize this by only fetching the required information.
status, err := clientAPIClient.Status(&apiclient.StatusArgs{
Patterns: []string{},
IncludeStorage: true,
})
if err != nil {
if strings.Contains(err.Error(), "filesystem for storage instance") || strings.Contains(err.Error(), "volume for storage instance") {
// Retry if we get this error. It means the storage is not ready yet.
return nil, &storageNotFoundError{input.AppName}
}
c.Errorf(err, "failed to get status")
return nil, err
}
var appStatus params.ApplicationStatus
Expand All @@ -827,6 +914,12 @@ func (c applicationsClient) ReadApplication(input *ReadApplicationInput) (*ReadA
return nil, fmt.Errorf("no status returned for application: %s", input.AppName)
}

storages := transformToStorageConstraints(status.Storage, status.Filesystems, status.Volumes)
// Print storage to console
for k, v := range storages {
c.Tracef("StorageConstraints constraints", map[string]interface{}{"storage": k, "constraints": v})
}

allocatedMachines := set.NewStrings()
for _, v := range appStatus.Units {
if v.Machine != "" {
Expand Down Expand Up @@ -993,6 +1086,7 @@ func (c applicationsClient) ReadApplication(input *ReadApplicationInput) (*ReadA
Principal: appInfo.Principal,
Placement: placement,
EndpointBindings: endpointBindings,
Storage: storages,
Resources: resourceRevisions,
}

Expand Down Expand Up @@ -1069,6 +1163,8 @@ func (c applicationsClient) UpdateApplication(input *UpdateApplicationInput) err
return err
}

setCharmConfig.StorageConstraints = input.StorageConstraints

err = applicationAPIClient.SetCharm(model.GenerationMaster, *setCharmConfig)
if err != nil {
return err
Expand Down
66 changes: 66 additions & 0 deletions internal/juju/applications_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,72 @@ func (s *ApplicationSuite) TestReadApplicationRetrySubordinate() {
s.Assert().Equal("[email protected]", resp.Base)
}

// TestReadApplicationRetryNotFoundStorageNotFoundError tests the case where the first response is a storage not found error.
// The second response is a real application.
func (s *ApplicationSuite) TestReadApplicationRetryNotFoundStorageNotFoundError() {
defer s.setupMocks(s.T()).Finish()
s.mockSharedClient.EXPECT().ModelType(gomock.Any()).Return(model.IAAS, nil).AnyTimes()

appName := "testapplication"
aExp := s.mockApplicationClient.EXPECT()

// First response is a storage not found error.
aExp.ApplicationsInfo(gomock.Any()).Return([]params.ApplicationInfoResult{{
Error: &params.Error{Message: `storage "testapplication" not found`, Code: "not found"},
}}, nil)

// Retry - expect ApplicationsInfo and Status to be called.
// The second time return a real application.
amdConst := constraints.MustParse("arch=amd64")
infoResult := params.ApplicationInfoResult{
Result: &params.ApplicationResult{
Tag: names.NewApplicationTag(appName).String(),
Charm: "ch:amd64/jammy/testcharm-5",
Base: params.Base{Name: "ubuntu", Channel: "22.04"},
Channel: "stable",
Constraints: amdConst,
Principal: true,
},
Error: nil,
}

aExp.ApplicationsInfo(gomock.Any()).Return([]params.ApplicationInfoResult{infoResult}, nil)
getResult := &params.ApplicationGetResults{
Application: appName,
CharmConfig: nil,
ApplicationConfig: nil,
Charm: "ch:amd64/jammy/testcharm-5",
Base: params.Base{Name: "ubuntu", Channel: "22.04"},
Channel: "stable",
Constraints: amdConst,
EndpointBindings: nil,
}
aExp.Get("master", appName).Return(getResult, nil)
statusResult := &params.FullStatus{
Applications: map[string]params.ApplicationStatus{appName: {
Charm: "ch:amd64/jammy/testcharm-5",
Units: map[string]params.UnitStatus{"testapplication/0": {
Machine: "0",
}},
}},
}
s.mockClient.EXPECT().Status(gomock.Any()).Return(statusResult, nil)

client := s.getApplicationsClient()
resp, err := client.ReadApplicationWithRetryOnNotFound(context.Background(),
&ReadApplicationInput{
ModelName: s.testModelName,
AppName: appName,
})
s.Require().NoError(err, "error from ReadApplicationWithRetryOnNotFound")
s.Require().NotNil(resp, "ReadApplicationWithRetryOnNotFound response")

s.Assert().Equal("testcharm", resp.Name)
s.Assert().Equal("stable", resp.Channel)
s.Assert().Equal(5, resp.Revision)
s.Assert().Equal("[email protected]", resp.Base)
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestApplicationSuite(t *testing.T) {
Expand Down
Loading

0 comments on commit f1a2dc9

Please sign in to comment.