Toaster basic support, defaults (#8)

* toasterdb poc

* toasterurl from envvar

* upd readme

* Toaster basic support, defaults

* Fix release only from master

* Upd readme

.github/workflows/release.yml

@@ -6,7 +6,6 @@ on:
       - 'master'
     tags:
       - 'v*'
-  pull_request:
 
 permissions:
   contents: write

README.md

@@ -47,15 +47,21 @@ Currently only `dimensions` with list of the required/expecting dimensions (from
 
 ## Inventory (dimensions) store
 
+You could set env variable `toasterurl` to point to TofuGu-Toaster, like `export toasterurl='https://accountid:[email protected]'`.
+Then TofuGu will connect and receive all the required dimension data from TofuGu-Toaster-ToasterDB.
+Additional parameter could be passed to tofugu `-w workspacename`. In general `workspacename` is the branch name of the source repo where the dimension is stored. If TofuGu-Toaster will not find dimension with specified `workspacename` it will try to return dimension from `master` workspace/branch!
+
 When you set dimensions in the tofugu flags `-d datacenter:staging1 `, tofugu will provide you inside code next variables:
 
 - var.tofugu_datacenter_name = will contain string `staging1`
 - var.tofugu_datacenter_manifest = will contain whole object from `staging1.json`
+- var.tofugu_datacenter_defaults = will contain whole object from `dim_defaults.json` IF file `dim_defaults.json` exists!
 
-[datacenter.json example in inventory](examples/inventory/demo-org/datacenter/staging1.json)
-
+Examples:
 
-[datacenter object from json used in code example](examples/tofies/demo-org/vpc/main.tf)
+- [staging1.json in Inventory Files](examples/inventory/demo-org/datacenter/staging1.json)
+- [dim_defaults.json in Inventory Files](examples/inventory/demo-org/datacenter/dim_defaults.json)
+- [datacenter object with defaults used in tf-code](examples/tofies/demo-org/vpc/main.tf#L5)
 
 ## Passing environment variables from shell
 
@@ -72,7 +78,7 @@ provider "aws" {
 }
 ```
 
-[Env variables used in code example](examples/tofies/demo-org/vpc/providers.tf)
+[Env variables used in code example](examples/tofies/demo-org/vpc/providers.tf#L3)
 
 ## $HOME/.tofugu
 
@@ -120,7 +126,7 @@ Other options contain hard-coded defaults:
 
 # Remote state in S3
 
-[Your terraform code (`tofi`) should contains at least:](examples/tofies/demo-org/vpc/versions.tf):
+[Your terraform code (`tofi`) should contains at least:](examples/tofies/demo-org/vpc/versions.tf#L4):
 ```
 terraform {
   backend "s3" {}

cmd/cook.go

@@ -30,6 +30,8 @@ var cookCmd = &cobra.Command{
 
 		tofuguStruct.TofiName, _ = cmd.Flags().GetString("tofi")
 		tofuguStruct.OrgName, _ = cmd.Flags().GetString("org")
+		tofuguStruct.Workspace, _ = cmd.Flags().GetString("workspace")
+		tofuguStruct.ToasterUrl = os.Getenv("toasterurl")
 		tofuguStruct.DimensionsFlags, _ = cmd.Flags().GetStringSlice("dimension")
 		tofuguStruct.TofiPath, _ = filepath.Abs(tofuguStruct.GetStringFromViperByOrgOrDefault("tofies_path") + "/" + tofuguStruct.OrgName + "/" + tofuguStruct.TofiName)
 		tofuguStruct.SharedModulesPath, _ = filepath.Abs(tofuguStruct.GetStringFromViperByOrgOrDefault("shared_modules_path"))
@@ -43,6 +45,7 @@ var cookCmd = &cobra.Command{
 		tofuguStruct.PrepareTemp()
 
 		tofuguStruct.GenerateVarsByDims()
+		tofuguStruct.GenerateVarsByDimOptional("defaults")
 		tofuguStruct.GenerateVarsByEnvVars()
 
 		//Local variables for child execution
@@ -113,6 +116,7 @@ func init() {
 	cookCmd.Flags().StringP("tofi", "t", "", "specify tofu unit")
 	//viper.BindPFlag("tofi", cookCmd.Flags().Lookup("tofi"))
 	cookCmd.Flags().StringP("org", "o", "", "specify org")
+	cookCmd.Flags().StringP("workspace", "w", "master", "specify workspace for toaster")
 	cookCmd.Flags().BoolP("clean", "c", false, "remove tmp after execution")
 	//viper.BindPFlag("org", cookCmd.Flags().Lookup("org"))
 	cookCmd.MarkFlagRequired("tofi")

examples/inventory/demo-org/application/first-app.json

@@ -0,0 +1,8 @@
+{
+    "services": {
+        "first-service": {
+            "cmd": "sh -c sleep 10",
+            "workDir": "app"
+        }
+    }
+}

examples/inventory/demo-org/datacenter/dim_defaults.json

@@ -0,0 +1,5 @@
+{
+    "test-account": {
+        "enable_dns_support": true
+    }
+}

examples/tofies/demo-org/vpc/main.tf

@@ -1,12 +1,13 @@
 //use shared-module
 module "vpc" {
   source = "./shared-modules/create_vpc"
-  cidr = var.tofugu_datacenter_manifest[var.tofugu_account_name].cidr
+  cidr = var.tofugu_datacenter_data[var.tofugu_account_name].cidr
+  enable_dns_support = try(var.tofugu_datacenter_data[var.tofugu_account_name].enable_dns_support, var.tofugu_datacenter_defaults[var.tofugu_account_name].enable_dns_support)
 }
 
 module "vpc_example_simple-vpc" {
   source  = "terraform-aws-modules/vpc/aws"
   version = "5.7.1"
 
-  cidr = var.tofugu_datacenter_manifest[var.tofugu_account_name].cidr
+  cidr = var.tofugu_datacenter_data[var.tofugu_account_name].cidr
 }

examples/tofies/demo-org/vpc/output.tf

@@ -3,5 +3,5 @@ output "region_from_env" {
 }
 
 output "region_from_inv" {
-    value = var.tofugu_account_manifest.region
+    value = var.tofugu_account_data.region
 }

examples/tofies/shared-modules/create_vpc/main.tf

@@ -1,3 +1,4 @@
 resource "aws_vpc" "example" {
-  cidr_block = var.cidr
+  cidr_block = var.cidr 
+  enable_dns_support = var.enable_dns_support
 }

examples/tofies/shared-modules/create_vpc/variables.tf

@@ -1,3 +1,7 @@
 variable "cidr" {
   type = string
+}
+
+variable "enable_dns_support" {
+  type = bool
 }

utils/dimensions.go

@@ -2,7 +2,6 @@ package utils
 
 import (
 	"log"
-	"os"
 	"strings"
 )
 
@@ -11,8 +10,7 @@ func (tofuguStruct *Tofugu) ParseDimensions() {
 
 	for _, dimension := range tofuguStruct.TofiManifest.Dimensions {
 		if _, ok := parsedDimArgs[dimension]; !ok {
-			log.Println("dimension " + dimension + " not passed with -d arg")
-			os.Exit(1)
+			log.Fatalln("dimension " + dimension + " not passed with -d arg")
 		}
 	}
 
@@ -23,6 +21,9 @@ func parseDimArgs(dimensionsArgs []string) map[string]string {
 	parsedDimArgs := make(map[string]string)
 	for _, dimension := range dimensionsArgs {
 		dimensionSlice := strings.SplitN(dimension, ":", 2)
+		if strings.HasPrefix(dimensionSlice[1], "dim_") {
+			log.Fatalln("dimension " + dimension + " with dim_ prefix can't be passed with -d arg")
+		}
 		parsedDimArgs[dimensionSlice[0]] = dimensionSlice[1]
 	}
 	return parsedDimArgs

utils/externals.go

@@ -3,6 +3,11 @@ package utils
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"os"
 
 	"github.com/spf13/viper"
 )
@@ -31,3 +36,62 @@ func (tofuguStruct *Tofugu) SetupStateS3Path() {
 	}
 	tofuguStruct.StateS3Path = stateS3Path + tofuguStruct.TofiName + ".tfstate"
 }
+
+func (tofuguStruct *Tofugu) GetDimData(dimensionKey string, dimensionValue string, skipOnNotFound bool) map[string]interface{} {
+	var dimensionJsonMap map[string]interface{}
+
+	if tofuguStruct.ToasterUrl == "" {
+		inventroyJsonPath := tofuguStruct.InventoryPath + "/" + dimensionKey + "/" + dimensionValue + ".json"
+		dimensionJsonBytes, err := os.ReadFile(inventroyJsonPath)
+		if err != nil {
+			if os.IsNotExist(err) && skipOnNotFound {
+				log.Println("TofuGu inventory files: Optional dimension " + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + " not found, skipping")
+				return dimensionJsonMap
+			}
+			log.Fatal("tofugu inventory files: error when opening dim file: ", err.Error())
+		}
+		err = json.Unmarshal(dimensionJsonBytes, &dimensionJsonMap)
+		if err != nil {
+			log.Fatal("tofugu error during Unmarshal(): ", err)
+		}
+	} else {
+		resp, err := http.Get(tofuguStruct.ToasterUrl + "/api/dimension/" + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + "?workspace=" + tofuguStruct.Workspace + "&fallbacktomaster=true")
+		if err != nil {
+			log.Fatalf("tofugu toaster: request Failed: %s", err)
+		} else if resp.StatusCode == 404 {
+			resp.Body.Close()
+			if skipOnNotFound {
+				log.Println("TofuGu Toaster: optional dimension " + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + " not found, skipping")
+				return dimensionJsonMap
+			} else {
+				log.Fatalln("tofugu toaster: dimension " + tofuguStruct.OrgName + "/" + dimensionKey + "/" + dimensionValue + " not found")
+			}
+		} else if resp.StatusCode != 200 {
+			resp.Body.Close()
+			log.Fatalf("tofugu toaster: request "+tofuguStruct.OrgName+"/"+dimensionKey+"/"+dimensionValue+"?workspace="+tofuguStruct.Workspace+" failed with response: %v", resp.StatusCode)
+		}
+		defer resp.Body.Close()
+
+		dimensionJsonBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			log.Fatalf("tofugu toaster: reading body response failed: %s", err)
+		}
+
+		var toasterResponse ToasterResponse
+		err = json.Unmarshal(dimensionJsonBytes, &toasterResponse)
+		if err != nil {
+			log.Fatal("tofugu toaster: error during unmarshal json response: ", err)
+		}
+
+		if len(toasterResponse.Dimensions) != 1 {
+			log.Fatalf("tofugu toaster: should be only one dimension in response")
+		}
+		if toasterResponse.Error != "" {
+			log.Println("TofuGu Toaster: " + toasterResponse.Error)
+		}
+		dimensionJsonMap = toasterResponse.Dimensions[0].DimData
+
+	}
+
+	return dimensionJsonMap
+}

utils/generatevars.go

@@ -9,24 +9,30 @@ import (
 
 func (tofuguStruct *Tofugu) GenerateVarsByDims() {
 	for dimKey, dimValue := range tofuguStruct.ParsedDimensions {
-		var inventroyJsonMap map[string]interface{}
-
-		inventroyJsonPath := tofuguStruct.InventoryPath + "/" + dimKey + "/" + dimValue + ".json"
-
-		inventroyJsonBytes, err := os.ReadFile(inventroyJsonPath)
-		if err != nil {
-			log.Fatal("Error when opening file: ", err)
-		}
-		json.Unmarshal(inventroyJsonBytes, &inventroyJsonMap)
+		dimensionJsonMap := tofuguStruct.GetDimData(dimKey, dimValue, false)
 
 		targetAutoTfvarMap := map[string]interface{}{
-			"tofugu_" + dimKey + "_manifest": inventroyJsonMap,
-			"tofugu_" + dimKey + "_name":     dimValue,
+			"tofugu_" + dimKey + "_data": dimensionJsonMap,
+			"tofugu_" + dimKey + "_name": dimValue,
 		}
 
 		writeTfvarsMaps(targetAutoTfvarMap, dimKey, tofuguStruct.CmdWorkTempDir)
-		log.Println("TofuGu generated tfvars for dimension: " + dimKey)
+		log.Println("TofuGu attached dimension in var.tofugu_" + dimKey + "_data and var.tofugu_" + dimKey + "_name")
+
+	}
+}
+
+func (tofuguStruct *Tofugu) GenerateVarsByDimOptional(optionType string) {
+	for dimKey := range tofuguStruct.ParsedDimensions {
+		dimensionJsonMap := tofuguStruct.GetDimData(dimKey, "dim_"+optionType, true)
+		if len(dimensionJsonMap) > 0 {
+			targetAutoTfvarMap := map[string]interface{}{
+				"tofugu_" + dimKey + "_" + optionType: dimensionJsonMap,
+			}
 
+			writeTfvarsMaps(targetAutoTfvarMap, dimKey+"_"+optionType, tofuguStruct.CmdWorkTempDir)
+			log.Println("TofuGu attached " + optionType + " in var.tofugu_" + dimKey + "_" + optionType)
+		}
 	}
 }
 
@@ -37,12 +43,12 @@ func (tofuguStruct *Tofugu) GenerateVarsByEnvVars() {
 		if strings.HasPrefix(envVar, "tofugu_envvar_") {
 			envVarList := strings.SplitN(envVar, "=", 2)
 			targetAutoTfvarMap[envVarList[0]] = envVarList[1]
+			log.Println("TofuGu attached env variable in var." + envVarList[0])
 		}
 	}
 
 	if len(targetAutoTfvarMap) > 0 {
 		writeTfvarsMaps(targetAutoTfvarMap, "envivars", tofuguStruct.CmdWorkTempDir)
-		log.Println("TofuGu generated tfvars for env variables")
 	}
 }
 
@@ -71,6 +77,12 @@ func writeTfvarsMaps(targetAutoTfvarMap map[string]interface{}, fileName string,
 }
 
 func marshalJsonAndWrite(jsonMap map[string]interface{}, jsonPath string) {
-	targetAutoTfvarMapBytes, _ := json.Marshal(jsonMap)
-	os.WriteFile(jsonPath, targetAutoTfvarMapBytes, os.ModePerm)
+	targetAutoTfvarMapBytes, err := json.Marshal(jsonMap)
+	if err != nil {
+		log.Fatal("tofugu failed to marshal json: ", err)
+	}
+	err = os.WriteFile(jsonPath, targetAutoTfvarMapBytes, os.ModePerm)
+	if err != nil {
+		log.Fatal("tofugu error writing file: ", err)
+	}
 }

utils/structures.go

@@ -12,8 +12,21 @@ type Tofugu struct {
 	CmdWorkTempDir    string
 	TofiManifest      tofiManifestStruct
 	StateS3Path       string
+	ToasterUrl        string
+	Workspace         string
 }
 
 type tofiManifestStruct struct {
 	Dimensions []string
 }
+
+type ToasterResponse struct {
+	Error      string
+	Dimensions []DimensionInToaster
+}
+
+type DimensionInToaster struct {
+	ID        string
+	WorkSpace string
+	DimData   map[string]interface{}
+}

utils/tofimanifest.go

@@ -11,14 +11,14 @@ func (tofuguStruct *Tofugu) ParseTofiManifest(tofiManifestFileName string) {
 	// Let's first read the `config.json` file
 	content, err := os.ReadFile(tofiManifestPath)
 	if err != nil {
-		log.Fatal("Error when opening file: ", err)
+		log.Fatal("tofugu error when opening file: ", err)
 	}
 
 	// Now let's unmarshall the data into `payload`
 	var tofiManifest tofiManifestStruct
 	err = json.Unmarshal(content, &tofiManifest)
 	if err != nil {
-		log.Fatal("Error during Unmarshal(): ", err)
+		log.Fatal("tofugu error during Unmarshal(): ", err)
 	}
 
 	tofuguStruct.TofiManifest = tofiManifest