From fda3726e0de3979e75a0132a4cae1a177f627251 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:40:29 +0200 Subject: [PATCH] ESS Staging integ test improvements (#3547) (#3604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * support tags on integration tests ESS deployments (cherry picked from commit be04fba8e3797607dcf5f7eaa238050a711d99a7) Co-authored-by: Paolo Chilà --- .../create_deployment_csp_configuration.yaml | 15 + .../ess/create_deployment_request.tmpl.json | 102 +++++++ pkg/testing/ess/deployment.go | 289 ++++-------------- pkg/testing/ess/provisioner.go | 28 +- 4 files changed, 206 insertions(+), 228 deletions(-) create mode 100644 pkg/testing/ess/create_deployment_csp_configuration.yaml create mode 100644 pkg/testing/ess/create_deployment_request.tmpl.json diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml new file mode 100644 index 00000000000..199f664a65a --- /dev/null +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -0,0 +1,15 @@ +gcp: + integrations_server_conf_id: "gcp.integrationsserver.n2.68x32x45.2" + elasticsearch_conf_id: "gcp.es.datahot.n2.68x10x45" + elasticsearch_deployment_template_id: "gcp-storage-optimized-v5" + kibana_instance_configuration_id: "gcp.kibana.n2.68x32x45" +azure: + integrations_server_conf_id: "azure.integrationsserver.fsv2.2" + elasticsearch_conf_id: "azure.es.datahot.edsv4" + elasticsearch_deployment_template_id: "azure-storage-optimized-v2" + kibana_instance_configuration_id: "azure.kibana.fsv2" +aws: + integrations_server_conf_id: "aws.integrationsserver.c5d.2.1" + elasticsearch_conf_id: "aws.es.datahot.i3.1.1" + elasticsearch_deployment_template_id: "aws-storage-optimized-v5" + kibana_instance_configuration_id: "aws.kibana.c5d.1.1" \ No newline at end of file diff --git a/pkg/testing/ess/create_deployment_request.tmpl.json b/pkg/testing/ess/create_deployment_request.tmpl.json new file mode 100644 index 00000000000..3ef93868708 --- /dev/null +++ b/pkg/testing/ess/create_deployment_request.tmpl.json @@ -0,0 +1,102 @@ +{ + "resources": { + "integrations_server": [ + { + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "region": "{{ .request.Region }}", + "plan": { + "cluster_topology": [ + { + "instance_configuration_id": "{{ .integrations_server_conf_id }}", + "zone_count": 1, + "size": { + "resource": "memory", + "value": 1024 + } + } + ], + "integrations_server": { + "version": "{{ .request.Version }}" + } + }, + "ref_id": "main-integrations_server" + } + ], + "elasticsearch": [ + { + "region": "{{ .request.Region }}", + "settings": { + "dedicated_masters_threshold": 6 + }, + "plan": { + "cluster_topology": [ + { + "zone_count": 1, + "elasticsearch": { + "node_attributes": { + "data": "hot" + } + }, + "instance_configuration_id": "{{.elasticsearch_conf_id}}", + "node_roles": [ + "master", + "ingest", + "transform", + "data_hot", + "remote_cluster_client", + "data_content" + ], + "id": "hot_content", + "size": { + "resource": "memory", + "value": 8192 + } + } + ], + "elasticsearch": { + "version": "{{ .request.Version }}", + "enabled_built_in_plugins": [] + }, + "deployment_template": { + "id": "{{ .elasticsearch_deployment_template_id }}" + } + }, + "ref_id": "main-elasticsearch" + } + ], + "enterprise_search": [], + "kibana": [ + { + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "region": "{{ .request.Region }}", + "plan": { + "cluster_topology": [ + { + "instance_configuration_id": "{{.kibana_instance_configuration_id}}", + "zone_count": 1, + "size": { + "resource": "memory", + "value": 1024 + } + } + ], + "kibana": { + "version": "{{ .request.Version }}", + "user_settings_json": { + "xpack.fleet.enableExperimental": ["agentTamperProtectionEnabled"] + } + } + }, + "ref_id": "main-kibana" + } + ] + }, + "settings": { + "autoscaling_enabled": false + }, + "name": "{{ .request.Name }}", + "metadata": { + "system_owned": false, + "tags": {{ json .request.Tags }} + } +} \ No newline at end of file diff --git a/pkg/testing/ess/deployment.go b/pkg/testing/ess/deployment.go index 9d9469036b0..a79f8cb58cb 100644 --- a/pkg/testing/ess/deployment.go +++ b/pkg/testing/ess/deployment.go @@ -7,19 +7,28 @@ package ess import ( "bytes" "context" + _ "embed" "encoding/json" "fmt" - "html/template" "net/http" "net/url" "strings" + "text/template" "time" + + "gopkg.in/yaml.v2" ) +type Tag struct { + Key string `json:"key"` + Value string `json:"value"` +} + type CreateDeploymentRequest struct { Name string `json:"name"` Region string `json:"region"` Version string `json:"version"` + Tags []Tag `json:"tags"` } type CreateDeploymentResponse struct { @@ -85,21 +94,16 @@ type DeploymentStatusResponse struct { // CreateDeployment creates the deployment with the specified configuration. func (c *Client) CreateDeployment(ctx context.Context, req CreateDeploymentRequest) (*CreateDeploymentResponse, error) { - tpl, err := deploymentTemplateFactory(req) + reqBodyBytes, err := generateCreateDeploymentRequestBody(req) if err != nil { return nil, err } - var buf bytes.Buffer - if err := tpl.Execute(&buf, req); err != nil { - return nil, fmt.Errorf("unable to create deployment creation request body: %w", err) - } - createResp, err := c.doPost( ctx, "deployments", "application/json", - &buf, + bytes.NewReader(reqBodyBytes), ) if err != nil { return nil, fmt.Errorf("error calling deployment creation API: %w", err) @@ -308,233 +312,70 @@ func overallStatus(statuses ...DeploymentStatus) DeploymentStatus { return overallStatus } -func deploymentTemplateFactory(req CreateDeploymentRequest) (*template.Template, error) { +//go:embed create_deployment_request.tmpl.json +var createDeploymentRequestTemplate string + +//go:embed create_deployment_csp_configuration.yaml +var cloudProviderSpecificValues []byte + +func generateCreateDeploymentRequestBody(req CreateDeploymentRequest) ([]byte, error) { regionParts := strings.Split(req.Region, "-") if len(regionParts) < 2 { return nil, fmt.Errorf("unable to parse CSP out of region [%s]", req.Region) } csp := regionParts[0] - var tplStr string - switch csp { - case "gcp": - tplStr = createDeploymentRequestTemplateGCP - case "azure": - tplStr = createDeploymentRequestTemplateAzure - default: - return nil, fmt.Errorf("unsupported CSP [%s]", csp) + templateContext, err := createDeploymentTemplateContext(csp, req) + if err != nil { + return nil, fmt.Errorf("creating request template context: %w", err) } - tpl, err := template.New("create_deployment_request").Parse(tplStr) + tpl, err := template.New("create_deployment_request"). + Funcs(template.FuncMap{"json": jsonMarshal}). + Parse(createDeploymentRequestTemplate) if err != nil { return nil, fmt.Errorf("unable to parse deployment creation template: %w", err) } - return tpl, nil + var bBuf bytes.Buffer + err = tpl.Execute(&bBuf, templateContext) + if err != nil { + return nil, fmt.Errorf("rendering create deployment request template with context %v : %w", templateContext, err) + } + return bBuf.Bytes(), nil } -const createDeploymentRequestTemplateGCP = ` -{ - "resources": { - "integrations_server": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "gcp.integrationsserver.n2.68x32x45.2", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "integrations_server": { - "version": "{{ .Version }}" - } - }, - "ref_id": "main-integrations_server" - } - ], - "elasticsearch": [ - { - "region": "{{ .Region }}", - "settings": { - "dedicated_masters_threshold": 6 - }, - "plan": { - "cluster_topology": [ - { - "zone_count": 1, - "elasticsearch": { - "node_attributes": { - "data": "hot" - } - }, - "instance_configuration_id": "gcp.es.datahot.n2.68x10x45", - "node_roles": [ - "master", - "ingest", - "transform", - "data_hot", - "remote_cluster_client", - "data_content" - ], - "id": "hot_content", - "size": { - "resource": "memory", - "value": 8192 - } - } - ], - "elasticsearch": { - "version": "{{ .Version }}", - "enabled_built_in_plugins": [] - }, - "deployment_template": { - "id": "gcp-storage-optimized-v5" - } - }, - "ref_id": "main-elasticsearch" - } - ], - "enterprise_search": [], - "kibana": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "gcp.kibana.n2.68x32x45", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "kibana": { - "version": "{{ .Version }}", - "user_settings_json": { - "xpack.fleet.enableExperimental": ["agentTamperProtectionEnabled"] - } - } - }, - "ref_id": "main-kibana" - } - ] - }, - "settings": { - "autoscaling_enabled": false - }, - "name": "{{ .Name }}", - "metadata": { - "system_owned": false - } -}` - -const createDeploymentRequestTemplateAzure = ` -{ - "resources": { - "integrations_server": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "azure.integrationsserver.fsv2.2", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "integrations_server": { - "version": "{{ .Version }}" - } - }, - "ref_id": "main-integrations_server" - } - ], - "elasticsearch": [ - { - "region": "{{ .Region }}", - "settings": { - "dedicated_masters_threshold": 6 - }, - "plan": { - "cluster_topology": [ - { - "zone_count": 1, - "elasticsearch": { - "node_attributes": { - "data": "hot" - } - }, - "instance_configuration_id": "azure.es.datahot.edsv4", - "node_roles": [ - "master", - "ingest", - "transform", - "data_hot", - "remote_cluster_client", - "data_content" - ], - "id": "hot_content", - "size": { - "resource": "memory", - "value": 8192 - } - } - ], - "elasticsearch": { - "version": "{{ .Version }}", - "enabled_built_in_plugins": [] - }, - "deployment_template": { - "id": "azure-storage-optimized-v2" - } - }, - "ref_id": "main-elasticsearch" - } - ], - "enterprise_search": [], - "kibana": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "azure.kibana.fsv2", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "kibana": { - "version": "{{ .Version }}", - "user_settings_json": { - "xpack.fleet.enableExperimental": ["agentTamperProtectionEnabled"] - } - } - }, - "ref_id": "main-kibana" - } - ] - }, - "settings": { - "autoscaling_enabled": false - }, - "name": "{{ .Name }}", - "metadata": { - "system_owned": false - } -}` +func jsonMarshal(in any) (string, error) { + jsonBytes, err := json.Marshal(in) + if err != nil { + return "", err + } + + return string(jsonBytes), nil +} + +func createDeploymentTemplateContext(csp string, req CreateDeploymentRequest) (map[string]any, error) { + cspSpecificContext, err := loadCspValues(csp) + if err != nil { + return nil, fmt.Errorf("loading csp-specific values for %q: %w", csp, err) + } + + cspSpecificContext["request"] = req + + return cspSpecificContext, nil +} + +func loadCspValues(csp string) (map[string]any, error) { + var cspValues map[string]map[string]any + + err := yaml.Unmarshal(cloudProviderSpecificValues, &cspValues) + if err != nil { + return nil, fmt.Errorf("unmarshalling error: %w", err) + } + values, supportedCSP := cspValues[csp] + if !supportedCSP { + return nil, fmt.Errorf("csp %s not supported", csp) + } + + return values, nil +} diff --git a/pkg/testing/ess/provisioner.go b/pkg/testing/ess/provisioner.go index 941cf5bcaf7..081b4100869 100644 --- a/pkg/testing/ess/provisioner.go +++ b/pkg/testing/ess/provisioner.go @@ -67,7 +67,14 @@ func (p *provisioner) Provision(ctx context.Context, requests []runner.StackRequ for _, r := range requests { // allow up to 2 minutes for each create request createCtx, createCancel := context.WithTimeout(ctx, 2*time.Minute) - resp, err := p.createDeployment(createCtx, r) + resp, err := p.createDeployment(createCtx, r, + map[string]string{ + "division": "engineering", + "org": "ingest", + "team": "elastic-agent", + "project": "elastic-agent", + "integration-tests": "true", + }) createCancel() if err != nil { return nil, err @@ -131,17 +138,30 @@ func (p *provisioner) Clean(ctx context.Context, stacks []runner.Stack) error { return nil } -func (p *provisioner) createDeployment(ctx context.Context, r runner.StackRequest) (*CreateDeploymentResponse, error) { +func (p *provisioner) createDeployment(ctx context.Context, r runner.StackRequest, tags map[string]string) (*CreateDeploymentResponse, error) { ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() p.logger.Logf("Creating stack %s (%s)", r.Version, r.ID) name := fmt.Sprintf("%s-%s", strings.Replace(p.cfg.Identifier, ".", "-", -1), r.ID) - resp, err := p.client.CreateDeployment(ctx, CreateDeploymentRequest{ + + // prepare tags + tagArray := make([]Tag, 0, len(tags)) + for k, v := range tags { + tagArray = append(tagArray, Tag{ + Key: k, + Value: v, + }) + } + + createDeploymentRequest := CreateDeploymentRequest{ Name: name, Region: p.cfg.Region, Version: r.Version, - }) + Tags: tagArray, + } + + resp, err := p.client.CreateDeployment(ctx, createDeploymentRequest) if err != nil { p.logger.Logf("Failed to create ESS cloud %s: %s", r.Version, err) return nil, fmt.Errorf("failed to create ESS cloud for version %s: %w", r.Version, err)