From 9f6e004cf2027aabe4139536dfe1b1bd47e958ab Mon Sep 17 00:00:00 2001 From: Dima Altukhov Date: Mon, 29 Apr 2024 14:30:54 +0300 Subject: [PATCH] Toaster basic support, defaults (#8) * toasterdb poc * toasterurl from envvar * upd readme * Toaster basic support, defaults * Fix release only from master * Upd readme --- .github/workflows/release.yml | 1 - README.md | 16 +++-- cmd/cook.go | 4 ++ .../demo-org/application/first-app.json | 8 +++ .../demo-org/datacenter/dim_defaults.json | 5 ++ examples/tofies/demo-org/vpc/main.tf | 5 +- examples/tofies/demo-org/vpc/output.tf | 2 +- .../tofies/shared-modules/create_vpc/main.tf | 3 +- .../shared-modules/create_vpc/variables.tf | 4 ++ utils/dimensions.go | 7 +- utils/externals.go | 64 +++++++++++++++++++ utils/generatevars.go | 42 +++++++----- utils/structures.go | 13 ++++ utils/tofimanifest.go | 4 +- 14 files changed, 148 insertions(+), 30 deletions(-) create mode 100644 examples/inventory/demo-org/application/first-app.json create mode 100644 examples/inventory/demo-org/datacenter/dim_defaults.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8afe975..f98b448 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,6 @@ on: - 'master' tags: - 'v*' - pull_request: permissions: contents: write diff --git a/README.md b/README.md index a93d6df..2916233 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,21 @@ Currently only `dimensions` with list of the required/expecting dimensions (from ## Inventory (dimensions) store +You could set env variable `toasterurl` to point to TofuGu-Toaster, like `export toasterurl='https://accountid:accountpass@toaster.example.com'`. +Then TofuGu will connect and receive all the required dimension data from TofuGu-Toaster-ToasterDB. +Additional parameter could be passed to tofugu `-w workspacename`. In general `workspacename` is the branch name of the source repo where the dimension is stored. If TofuGu-Toaster will not find dimension with specified `workspacename` it will try to return dimension from `master` workspace/branch! + When you set dimensions in the tofugu flags `-d datacenter:staging1 `, tofugu will provide you inside code next variables: - var.tofugu_datacenter_name = will contain string `staging1` - var.tofugu_datacenter_manifest = will contain whole object from `staging1.json` +- var.tofugu_datacenter_defaults = will contain whole object from `dim_defaults.json` IF file `dim_defaults.json` exists! -[datacenter.json example in inventory](examples/inventory/demo-org/datacenter/staging1.json) - +Examples: -[datacenter object from json used in code example](examples/tofies/demo-org/vpc/main.tf) +- [staging1.json in Inventory Files](examples/inventory/demo-org/datacenter/staging1.json) +- [dim_defaults.json in Inventory Files](examples/inventory/demo-org/datacenter/dim_defaults.json) +- [datacenter object with defaults used in tf-code](examples/tofies/demo-org/vpc/main.tf#L5) ## Passing environment variables from shell @@ -72,7 +78,7 @@ provider "aws" { } ``` -[Env variables used in code example](examples/tofies/demo-org/vpc/providers.tf) +[Env variables used in code example](examples/tofies/demo-org/vpc/providers.tf#L3) ## $HOME/.tofugu @@ -120,7 +126,7 @@ Other options contain hard-coded defaults: # Remote state in S3 -[Your terraform code (`tofi`) should contains at least:](examples/tofies/demo-org/vpc/versions.tf): +[Your terraform code (`tofi`) should contains at least:](examples/tofies/demo-org/vpc/versions.tf#L4): ``` terraform { backend "s3" {} diff --git a/cmd/cook.go b/cmd/cook.go index 07ed565..08e4b16 100644 --- a/cmd/cook.go +++ b/cmd/cook.go @@ -30,6 +30,8 @@ var cookCmd = &cobra.Command{ tofuguStruct.TofiName, _ = cmd.Flags().GetString("tofi") tofuguStruct.OrgName, _ = cmd.Flags().GetString("org") + tofuguStruct.Workspace, _ = cmd.Flags().GetString("workspace") + tofuguStruct.ToasterUrl = os.Getenv("toasterurl") tofuguStruct.DimensionsFlags, _ = cmd.Flags().GetStringSlice("dimension") tofuguStruct.TofiPath, _ = filepath.Abs(tofuguStruct.GetStringFromViperByOrgOrDefault("tofies_path") + "/" + tofuguStruct.OrgName + "/" + tofuguStruct.TofiName) tofuguStruct.SharedModulesPath, _ = filepath.Abs(tofuguStruct.GetStringFromViperByOrgOrDefault("shared_modules_path")) @@ -43,6 +45,7 @@ var cookCmd = &cobra.Command{ tofuguStruct.PrepareTemp() tofuguStruct.GenerateVarsByDims() + tofuguStruct.GenerateVarsByDimOptional("defaults") tofuguStruct.GenerateVarsByEnvVars() //Local variables for child execution @@ -113,6 +116,7 @@ func init() { cookCmd.Flags().StringP("tofi", "t", "", "specify tofu unit") //viper.BindPFlag("tofi", cookCmd.Flags().Lookup("tofi")) cookCmd.Flags().StringP("org", "o", "", "specify org") + cookCmd.Flags().StringP("workspace", "w", "master", "specify workspace for toaster") cookCmd.Flags().BoolP("clean", "c", false, "remove tmp after execution") //viper.BindPFlag("org", cookCmd.Flags().Lookup("org")) cookCmd.MarkFlagRequired("tofi") diff --git a/examples/inventory/demo-org/application/first-app.json b/examples/inventory/demo-org/application/first-app.json new file mode 100644 index 0000000..9abcbca --- /dev/null +++ b/examples/inventory/demo-org/application/first-app.json @@ -0,0 +1,8 @@ +{ + "services": { + "first-service": { + "cmd": "sh -c sleep 10", + "workDir": "app" + } + } +} \ No newline at end of file diff --git a/examples/inventory/demo-org/datacenter/dim_defaults.json b/examples/inventory/demo-org/datacenter/dim_defaults.json new file mode 100644 index 0000000..bd1c299 --- /dev/null +++ b/examples/inventory/demo-org/datacenter/dim_defaults.json @@ -0,0 +1,5 @@ +{ + "test-account": { + "enable_dns_support": true + } +} \ No newline at end of file diff --git a/examples/tofies/demo-org/vpc/main.tf b/examples/tofies/demo-org/vpc/main.tf index b4734ce..e8a47b4 100644 --- a/examples/tofies/demo-org/vpc/main.tf +++ b/examples/tofies/demo-org/vpc/main.tf @@ -1,12 +1,13 @@ //use shared-module module "vpc" { source = "./shared-modules/create_vpc" - cidr = var.tofugu_datacenter_manifest[var.tofugu_account_name].cidr + cidr = var.tofugu_datacenter_data[var.tofugu_account_name].cidr + enable_dns_support = try(var.tofugu_datacenter_data[var.tofugu_account_name].enable_dns_support, var.tofugu_datacenter_defaults[var.tofugu_account_name].enable_dns_support) } module "vpc_example_simple-vpc" { source = "terraform-aws-modules/vpc/aws" version = "5.7.1" - cidr = var.tofugu_datacenter_manifest[var.tofugu_account_name].cidr + cidr = var.tofugu_datacenter_data[var.tofugu_account_name].cidr } \ No newline at end of file diff --git a/examples/tofies/demo-org/vpc/output.tf b/examples/tofies/demo-org/vpc/output.tf index 543ceb9..f8950ab 100644 --- a/examples/tofies/demo-org/vpc/output.tf +++ b/examples/tofies/demo-org/vpc/output.tf @@ -3,5 +3,5 @@ output "region_from_env" { } output "region_from_inv" { - value = var.tofugu_account_manifest.region + value = var.tofugu_account_data.region } \ No newline at end of file diff --git a/examples/tofies/shared-modules/create_vpc/main.tf b/examples/tofies/shared-modules/create_vpc/main.tf index b7ffb57..573eda1 100644 --- a/examples/tofies/shared-modules/create_vpc/main.tf +++ b/examples/tofies/shared-modules/create_vpc/main.tf @@ -1,3 +1,4 @@ resource "aws_vpc" "example" { - cidr_block = var.cidr + cidr_block = var.cidr + enable_dns_support = var.enable_dns_support } \ No newline at end of file diff --git a/examples/tofies/shared-modules/create_vpc/variables.tf b/examples/tofies/shared-modules/create_vpc/variables.tf index 11f94c3..5e337c5 100644 --- a/examples/tofies/shared-modules/create_vpc/variables.tf +++ b/examples/tofies/shared-modules/create_vpc/variables.tf @@ -1,3 +1,7 @@ variable "cidr" { type = string +} + +variable "enable_dns_support" { + type = bool } \ No newline at end of file diff --git a/utils/dimensions.go b/utils/dimensions.go index 6ea97b4..9e69be0 100644 --- a/utils/dimensions.go +++ b/utils/dimensions.go @@ -2,7 +2,6 @@ package utils import ( "log" - "os" "strings" ) @@ -11,8 +10,7 @@ func (tofuguStruct *Tofugu) ParseDimensions() { for _, dimension := range tofuguStruct.TofiManifest.Dimensions { if _, ok := parsedDimArgs[dimension]; !ok { - log.Println("dimension " + dimension + " not passed with -d arg") - os.Exit(1) + log.Fatalln("dimension " + dimension + " not passed with -d arg") } } @@ -23,6 +21,9 @@ func parseDimArgs(dimensionsArgs []string) map[string]string { parsedDimArgs := make(map[string]string) for _, dimension := range dimensionsArgs { dimensionSlice := strings.SplitN(dimension, ":", 2) + if strings.HasPrefix(dimensionSlice[1], "dim_") { + log.Fatalln("dimension " + dimension + " with dim_ prefix can't be passed with -d arg") + } parsedDimArgs[dimensionSlice[0]] = dimensionSlice[1] } return parsedDimArgs diff --git a/utils/externals.go b/utils/externals.go index b60c1d0..1dbb676 100644 --- a/utils/externals.go +++ b/utils/externals.go @@ -3,6 +3,11 @@ package utils import ( "crypto/md5" "encoding/hex" + "encoding/json" + "io" + "log" + "net/http" + "os" "github.com/spf13/viper" ) @@ -31,3 +36,62 @@ func (tofuguStruct *Tofugu) SetupStateS3Path() { } tofuguStruct.StateS3Path = stateS3Path + tofuguStruct.TofiName + ".tfstate" } + +func (tofuguStruct *Tofugu) GetDimData(dimensionKey string, dimensionValue string, skipOnNotFound bool) map[string]interface{} { + var dimensionJsonMap map[string]interface{} + + if tofuguStruct.ToasterUrl == "" { + inventroyJsonPath := tofuguStruct.InventoryPath + "/" + dimensionKey + "/" + dimensionValue + ".json" + dimensionJsonBytes, err := os.ReadFile(inventroyJsonPath) + if err != nil { + if os.IsNotExist(err) && skipOnNotFound { + log.Println("TofuGu inventory files: Optional dimension " + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + " not found, skipping") + return dimensionJsonMap + } + log.Fatal("tofugu inventory files: error when opening dim file: ", err.Error()) + } + err = json.Unmarshal(dimensionJsonBytes, &dimensionJsonMap) + if err != nil { + log.Fatal("tofugu error during Unmarshal(): ", err) + } + } else { + resp, err := http.Get(tofuguStruct.ToasterUrl + "/api/dimension/" + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + "?workspace=" + tofuguStruct.Workspace + "&fallbacktomaster=true") + if err != nil { + log.Fatalf("tofugu toaster: request Failed: %s", err) + } else if resp.StatusCode == 404 { + resp.Body.Close() + if skipOnNotFound { + log.Println("TofuGu Toaster: optional dimension " + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + " not found, skipping") + return dimensionJsonMap + } else { + log.Fatalln("tofugu toaster: dimension " + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + " not found") + } + } else if resp.StatusCode != 200 { + resp.Body.Close() + log.Fatalf("tofugu toaster: request "+tofuguStruct.OrgName+"/"+dimensionKey+"/"+dimensionValue+"?workspace="+tofuguStruct.Workspace+" failed with response: %v", resp.StatusCode) + } + defer resp.Body.Close() + + dimensionJsonBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatalf("tofugu toaster: reading body response failed: %s", err) + } + + var toasterResponse ToasterResponse + err = json.Unmarshal(dimensionJsonBytes, &toasterResponse) + if err != nil { + log.Fatal("tofugu toaster: error during unmarshal json response: ", err) + } + + if len(toasterResponse.Dimensions) != 1 { + log.Fatalf("tofugu toaster: should be only one dimension in response") + } + if toasterResponse.Error != "" { + log.Println("TofuGu Toaster: " + toasterResponse.Error) + } + dimensionJsonMap = toasterResponse.Dimensions[0].DimData + + } + + return dimensionJsonMap +} diff --git a/utils/generatevars.go b/utils/generatevars.go index 4af7a62..b627752 100644 --- a/utils/generatevars.go +++ b/utils/generatevars.go @@ -9,24 +9,30 @@ import ( func (tofuguStruct *Tofugu) GenerateVarsByDims() { for dimKey, dimValue := range tofuguStruct.ParsedDimensions { - var inventroyJsonMap map[string]interface{} - - inventroyJsonPath := tofuguStruct.InventoryPath + "/" + dimKey + "/" + dimValue + ".json" - - inventroyJsonBytes, err := os.ReadFile(inventroyJsonPath) - if err != nil { - log.Fatal("Error when opening file: ", err) - } - json.Unmarshal(inventroyJsonBytes, &inventroyJsonMap) + dimensionJsonMap := tofuguStruct.GetDimData(dimKey, dimValue, false) targetAutoTfvarMap := map[string]interface{}{ - "tofugu_" + dimKey + "_manifest": inventroyJsonMap, - "tofugu_" + dimKey + "_name": dimValue, + "tofugu_" + dimKey + "_data": dimensionJsonMap, + "tofugu_" + dimKey + "_name": dimValue, } writeTfvarsMaps(targetAutoTfvarMap, dimKey, tofuguStruct.CmdWorkTempDir) - log.Println("TofuGu generated tfvars for dimension: " + dimKey) + log.Println("TofuGu attached dimension in var.tofugu_" + dimKey + "_data and var.tofugu_" + dimKey + "_name") + + } +} + +func (tofuguStruct *Tofugu) GenerateVarsByDimOptional(optionType string) { + for dimKey := range tofuguStruct.ParsedDimensions { + dimensionJsonMap := tofuguStruct.GetDimData(dimKey, "dim_"+optionType, true) + if len(dimensionJsonMap) > 0 { + targetAutoTfvarMap := map[string]interface{}{ + "tofugu_" + dimKey + "_" + optionType: dimensionJsonMap, + } + writeTfvarsMaps(targetAutoTfvarMap, dimKey+"_"+optionType, tofuguStruct.CmdWorkTempDir) + log.Println("TofuGu attached " + optionType + " in var.tofugu_" + dimKey + "_" + optionType) + } } } @@ -37,12 +43,12 @@ func (tofuguStruct *Tofugu) GenerateVarsByEnvVars() { if strings.HasPrefix(envVar, "tofugu_envvar_") { envVarList := strings.SplitN(envVar, "=", 2) targetAutoTfvarMap[envVarList[0]] = envVarList[1] + log.Println("TofuGu attached env variable in var." + envVarList[0]) } } if len(targetAutoTfvarMap) > 0 { writeTfvarsMaps(targetAutoTfvarMap, "envivars", tofuguStruct.CmdWorkTempDir) - log.Println("TofuGu generated tfvars for env variables") } } @@ -71,6 +77,12 @@ func writeTfvarsMaps(targetAutoTfvarMap map[string]interface{}, fileName string, } func marshalJsonAndWrite(jsonMap map[string]interface{}, jsonPath string) { - targetAutoTfvarMapBytes, _ := json.Marshal(jsonMap) - os.WriteFile(jsonPath, targetAutoTfvarMapBytes, os.ModePerm) + targetAutoTfvarMapBytes, err := json.Marshal(jsonMap) + if err != nil { + log.Fatal("tofugu failed to marshal json: ", err) + } + err = os.WriteFile(jsonPath, targetAutoTfvarMapBytes, os.ModePerm) + if err != nil { + log.Fatal("tofugu error writing file: ", err) + } } diff --git a/utils/structures.go b/utils/structures.go index e2ad71d..bf36c4e 100644 --- a/utils/structures.go +++ b/utils/structures.go @@ -12,8 +12,21 @@ type Tofugu struct { CmdWorkTempDir string TofiManifest tofiManifestStruct StateS3Path string + ToasterUrl string + Workspace string } type tofiManifestStruct struct { Dimensions []string } + +type ToasterResponse struct { + Error string + Dimensions []DimensionInToaster +} + +type DimensionInToaster struct { + ID string + WorkSpace string + DimData map[string]interface{} +} diff --git a/utils/tofimanifest.go b/utils/tofimanifest.go index 98b75e2..83a8fbe 100644 --- a/utils/tofimanifest.go +++ b/utils/tofimanifest.go @@ -11,14 +11,14 @@ func (tofuguStruct *Tofugu) ParseTofiManifest(tofiManifestFileName string) { // Let's first read the `config.json` file content, err := os.ReadFile(tofiManifestPath) if err != nil { - log.Fatal("Error when opening file: ", err) + log.Fatal("tofugu error when opening file: ", err) } // Now let's unmarshall the data into `payload` var tofiManifest tofiManifestStruct err = json.Unmarshal(content, &tofiManifest) if err != nil { - log.Fatal("Error during Unmarshal(): ", err) + log.Fatal("tofugu error during Unmarshal(): ", err) } tofuguStruct.TofiManifest = tofiManifest