Skip to content

Commit

Permalink
ci: support skip nightly tests by distro/host
Browse files Browse the repository at this point in the history
  • Loading branch information
kb-newrelic committed Jan 27, 2025
1 parent a85d6f7 commit ede797c
Show file tree
Hide file tree
Showing 12 changed files with 128 additions and 50 deletions.
3 changes: 3 additions & 0 deletions distributions/nr-otel-collector/test-spec.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
nightly:
ec2:
enabled: true
42 changes: 24 additions & 18 deletions test/e2e/hostmetrics_nightly/hostmetrics_nightly_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,40 +10,46 @@ import (
"time"
)

type systemUnderTest struct {
hostNamePattern string
excludedMetrics []string
}

var ec2Ubuntu22 = systemUnderTest{
hostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "ec2_ubuntu22_04"),
var ec2Ubuntu22 = spec.NightlySystemUnderTest{
HostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "ec2_ubuntu22_04"),
// TODO: NR-362121
excludedMetrics: []string{"system.paging.usage"},
ExcludedMetrics: []string{"system.paging.usage"},
SkipIf: func(testSpec *spec.TestSpec) bool {
return !testSpec.Nightly.EC2.Enabled
},
}
var ec2Ubuntu24 = systemUnderTest{
hostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "ec2_ubuntu24_04"),
var ec2Ubuntu24 = spec.NightlySystemUnderTest{
HostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "ec2_ubuntu24_04"),
// TODO: NR-362121
excludedMetrics: []string{"system.paging.usage"},
ExcludedMetrics: []string{"system.paging.usage"},
SkipIf: func(testSpec *spec.TestSpec) bool {
return !testSpec.Nightly.EC2.Enabled
},
}
var k8sNode = systemUnderTest{
hostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "k8s_node"),
excludedMetrics: []string{"system.paging.usage"},
var k8sNode = spec.NightlySystemUnderTest{
HostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "k8s_node"),
ExcludedMetrics: []string{"system.paging.usage"},
}

func TestNightlyHostMetrics(t *testing.T) {
testutil.TagAsNightlyTest(t)
testSpec := spec.LoadTestSpec()

// space out requests to not run into 25 concurrent request limit
requestsPerSecond := 4.0
requestSpacing := time.Duration((1/requestsPerSecond)*1000) * time.Millisecond
client := nr.NewClient()

for _, sut := range []systemUnderTest{ec2Ubuntu22, ec2Ubuntu24, k8sNode} {
for i, testCase := range spec.GetOnHostTestCasesWithout(sut.excludedMetrics) {
t.Run(fmt.Sprintf("%s/%d/%s", sut.hostNamePattern, i, testCase.Name), func(t *testing.T) {
for _, sut := range []spec.NightlySystemUnderTest{ec2Ubuntu22, ec2Ubuntu24, k8sNode} {
if sut.SkipIf != nil && sut.SkipIf(testSpec) {
t.Logf("Skipping nightly system-under-test: %s", sut.HostNamePattern)
continue
}
for i, testCase := range spec.GetOnHostTestCasesWithout(sut.ExcludedMetrics) {
t.Run(fmt.Sprintf("%s/%d/%s", sut.HostNamePattern, i, testCase.Name), func(t *testing.T) {
t.Parallel()
assertionFactory := assert.NewNrMetricAssertionFactory(
fmt.Sprintf("WHERE host.name like '%s'", sut.hostNamePattern),
fmt.Sprintf("WHERE host.name like '%s'", sut.HostNamePattern),
"2 hour ago",
)
assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
Expand Down
15 changes: 2 additions & 13 deletions test/e2e/util/chart/chart.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package chart

import (
"fmt"
"os"
"path"
"strings"
testutil "test/e2e/util/test"
)

type Chart interface {
Expand All @@ -17,13 +14,5 @@ type Meta struct {
}

func (m Meta) ChartPath() string {
pwd, err := os.Getwd()
if err != nil {
panic(err)
}
testDirPath := path.Clean(fmt.Sprintf("%s/../..", pwd))
if !strings.HasSuffix(testDirPath, "test") {
panic(fmt.Sprintf("Unexpected directory structure: %s should be the test dir containing the charts directory (pwd: %s)", testDirPath, pwd))
}
return path.Join(testDirPath, "charts", m.name)
return testutil.NewPathRelativeToRootDir("test/charts/" + m.name)
}
5 changes: 0 additions & 5 deletions test/e2e/util/chart/mocked_backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,3 @@ func (m *MockedBackendChart) RequiredChartValues() map[string]string {
"image.tag": envutil.GetImageTag(),
}
}

func (m *MockedBackendChart) GetCollectorSelector() string {
//TODO implement me
panic("implement me")
}
7 changes: 7 additions & 0 deletions test/e2e/util/spec/nightly.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package spec

type NightlySystemUnderTest struct {
HostNamePattern string
ExcludedMetrics []string
SkipIf func(testSpec *TestSpec) bool
}
4 changes: 3 additions & 1 deletion test/e2e/util/spec/on_host.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package spec

import "test/e2e/util/assert"
import (
"test/e2e/util/assert"
)

type TestCase struct {
Name string
Expand Down
34 changes: 34 additions & 0 deletions test/e2e/util/spec/specfile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package spec

import (
"fmt"
"gopkg.in/yaml.v3"
"os"
envutil "test/e2e/util/env"
testutil "test/e2e/util/test"
)

type TestSpec struct {
Nightly struct {
EC2 struct {
Enabled bool `yaml:"enabled"`
} `yaml:"ec2"`
} `yaml:"nightly"`
}

func LoadTestSpec() *TestSpec {
distro := envutil.GetDistro()
testSpecFile := testutil.NewPathRelativeToRootDir("distributions/" + distro + "/test-spec.yaml")
testSpecData, err := os.ReadFile(testSpecFile)
if err != nil {
panic(fmt.Errorf("failed to read test spec file: %w", err))
}

var testSpec TestSpec
err = yaml.Unmarshal(testSpecData, &testSpec)
if err != nil {
panic(fmt.Errorf("failed to unmarshal test spec: %w", err))
}

return &testSpec
}
25 changes: 18 additions & 7 deletions test/e2e/util/test/hostname.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,28 @@
package test

import "fmt"
import (
"strings"
envutil "test/e2e/util/env"
)

const (
Wildcard = "%"
hostNameSegmentSeparator = "-"
)

func NewHostNamePrefix(envName string, deployId string, hostType string) string {
// TODO: incorporate distro into hostname when generalizing nightly to support multiple distro
distro := getNormalizedDistro()
// only a prefix as helm chart appends hostId
return fmt.Sprintf("%s-%s-%s", envName, deployId, hostType)
return strings.Join([]string{envName, deployId, distro, hostType}, hostNameSegmentSeparator)
}

const Wildcard = "%"

func NewNrQueryHostNamePattern(envName string, deployId string, hostType string) string {
// TODO: incorporate distro into hostname when generalizing nightly to support multiple distro
distro := getNormalizedDistro()
hostId := Wildcard
return fmt.Sprintf("%s-%s-%s-%s", envName, deployId, hostType, hostId)
return strings.Join([]string{envName, deployId, distro, hostType, hostId}, hostNameSegmentSeparator)
}

func getNormalizedDistro() string {
// solely to improve readability - no technical necessity
return strings.Replace(envutil.GetDistro(), hostNameSegmentSeparator, "_", -1)
}
25 changes: 25 additions & 0 deletions test/e2e/util/test/path.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package test

import (
"fmt"
"os"
"path"
"strings"
)

func NewPathRelativeToRootDir(pathFromRoot string) string {
pwd, err := os.Getwd()
if err != nil {
panic(err)
}
rootDir := path.Clean(fmt.Sprintf("%s/../../..", pwd))

if _, err := os.Stat(path.Join(rootDir, ".github")); err != nil {
panic(fmt.Errorf("unexpected directory structure: %s should be the root dir (pwd: %s) but encountered error %w", rootDir, pwd, err))
}
var result = rootDir
for _, segment := range strings.Split(pathFromRoot, "/") {
result = path.Join(result, segment)
}
return result
}
2 changes: 1 addition & 1 deletion test/terraform/modules/ec2/main.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
locals {
collector_reported_hostname_prefix = "${var.test_environment}-${var.deploy_id}"
collector_reported_hostname_prefix = "${var.test_environment}-${var.deploy_id}-${var.collector_distro}"
instance_config = [
{
hostname_suffix = "ec2_ubuntu22_04-0"
Expand Down
1 change: 0 additions & 1 deletion test/terraform/modules/ec2/vars.tf
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ variable "nr_ingest_key" {
variable "collector_distro" {
description = "Name of the distribution of NRDOT to install"
type = string
default = "nr-otel-collector"
}

variable "collector_version" {
Expand Down
15 changes: 11 additions & 4 deletions test/terraform/permanent/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ locals {
for _, v in fileset(path.module, "../../../distributions/**") :
regex("../../../distributions/([^/]*).*", dirname(v))
])))
test_specs = {
for distro in local.distros :
distro => yamldecode(file("${path.module}/../../../distributions/${distro}/test-spec.yaml"))
}
releases_bucket_name = "nr-releases"
required_permissions_boundary_arn_for_new_roles = "arn:aws:iam::${var.aws_account_id}:policy/resource-provisioner-boundary"
}
Expand All @@ -15,7 +19,7 @@ data "aws_vpc" "this" {
module "ci_e2e_cluster" {
source = "../modules/eks_cluster"

name = "aws-ci-e2etest"
name = "aws-ci-e2etest"
permission_boundary = local.required_permissions_boundary_arn_for_new_roles
}

Expand Down Expand Up @@ -82,17 +86,18 @@ resource "random_string" "deploy_id" {
}

resource "helm_release" "ci_e2e_nightly" {
for_each = local.distros
depends_on = [module.ci_e2e_cluster, module.ecr]

name = "ci-e2etest-nightly"
chart = "../../charts/nr_backend"

create_namespace = true
namespace = "nightly"
namespace = "nightly-${each.key}"

set {
name = "image.repository"
value = module.ecr["nr-otel-collector"].repository_url
value = module.ecr[each.key].repository_url
}

set {
Expand All @@ -117,13 +122,15 @@ resource "helm_release" "ci_e2e_nightly" {

set {
name = "collector.hostname"
value = "${var.test_environment}-${random_string.deploy_id.result}-k8s_node"
value = "${var.test_environment}-${random_string.deploy_id.result}-${each.key}-k8s_node"
}
}

module "ci_e2e_ec2" {
for_each = toset([for distro in local.distros : distro if local.test_specs[distro].nightly.ec2.enabled])
source = "../modules/ec2"
releases_bucket_name = local.releases_bucket_name
collector_distro = each.key
nr_ingest_key = var.nr_ingest_key
# reuse vpc to avoid having to pay for second NAT gateway for this simple use case
vpc_id = module.ci_e2e_cluster.eks_vpc_id
Expand Down

0 comments on commit ede797c

Please sign in to comment.