Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CreateLogGroup Integration Testing #381

Merged
merged 23 commits into from
Dec 12, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
reintroduces new test changes
  • Loading branch information
dchappa committed Dec 1, 2023
commit 3fa3e1a5f62046f8504e2dceebe42fbbe25f8e60
196 changes: 129 additions & 67 deletions test/cloudwatchlogs/publish_logs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,36 +24,67 @@ import (
)

const (
configOutputPath = "/opt/aws/amazon-cloudwatch-agent/bin/config.json"
logLineId1 = "foo"
logLineId2 = "bar"
logFilePath = "/tmp/cwagent_log_test.log" // TODO: not sure how well this will work on Windows
sleepForFlush = 20 * time.Second // default flush interval is 5 seconds
configPathAutoRemoval = "resources/config_auto_removal.json"
configOutputPath = "/opt/aws/amazon-cloudwatch-agent/bin/config.json"
logLineId1 = "foo"
logLineId2 = "bar"
logFilePath = "/tmp/cwagent_log_test.log" // TODO: not sure how well this will work on Windows
sleepForFlush = 20 * time.Second // default flush interval is 5 seconds
configPathAutoRemoval = "resources/config_auto_removal.json"
standardLogGroupClass = "STANDARD"
infrequentAccessLogGroupClass = "INFREQUENT_ACCESS"
)

var logLineIds = []string{logLineId1, logLineId2}
var (
logLineIds = []string{logLineId1, logLineId2}
instanceId = awsservice.GetInstanceId()
writeToCloudWatchTestParameters = []writeToCloudWatchTestInput{
{
testName: "Happy path",
iterations: 100,
numExpectedLogs: 200,
configPath: "resources/config_log.json",
},
{
testName: "Client-side log filtering",
iterations: 100,
numExpectedLogs: 100,
configPath: "resources/config_log_filter.json",
},
}
cloudWatchLogGroupClassTestParameters = []cloudWatchLogGroupClassTestInput{
{
testName: "Standard log config",
configPath: "resources/config_log.json",
logGroupName: instanceId,
logGroupClass: standardLogGroupClass,
},
{
testName: "Standard log config with standard class specification",
configPath: "resources/config_log_standard_access.json",
logGroupName: instanceId + "-standard",
logGroupClass: standardLogGroupClass,
},
{
testName: "Standard log config with Infrequent_access class specification",
configPath: "resources/config_log_infrequent_access.json",
logGroupName: instanceId + "-infrequent_access",
logGroupClass: infrequentAccessLogGroupClass,
},
}
)

type input struct {
type writeToCloudWatchTestInput struct {
testName string
iterations int
numExpectedLogs int
configPath string
}

var testParameters = []input{
{
testName: "Happy path",
iterations: 100,
numExpectedLogs: 200,
configPath: "resources/config_log.json",
},
{
testName: "Client-side log filtering",
iterations: 100,
numExpectedLogs: 100,
configPath: "resources/config_log_filter.json",
},
type cloudWatchLogGroupClassTestInput struct {
testName string
configPath string
logGroupName string
logGroupClass string
}

func init() {
Expand All @@ -65,7 +96,6 @@ func init() {
func TestWriteLogsToCloudWatch(t *testing.T) {
// this uses the {instance_id} placeholder in the agent configuration,
// so we need to determine the host's instance ID for validation
instanceId := awsservice.GetInstanceId()
log.Printf("Found instance id %s", instanceId)

defer awsservice.DeleteLogGroupAndStream(instanceId, instanceId)
Expand All @@ -77,7 +107,7 @@ func TestWriteLogsToCloudWatch(t *testing.T) {
defer f.Close()
defer os.Remove(logFilePath)

for _, param := range testParameters {
for _, param := range writeToCloudWatchTestParameters {
t.Run(param.testName, func(t *testing.T) {
common.DeleteFile(common.AgentLogFile)
common.TouchFile(common.AgentLogFile)
Expand Down Expand Up @@ -109,49 +139,6 @@ func TestWriteLogsToCloudWatch(t *testing.T) {
}
}

func autoRemovalTestCleanup() {
instanceId := awsservice.GetInstanceId()
awsservice.DeleteLogGroupAndStream(instanceId, instanceId)
paths, _ := filepath.Glob(logFilePath + "*")
for _, p := range paths {
_ = os.Remove(p)
}
}

// checkData queries CWL and verifies the number of log lines.
func checkData(t *testing.T, start time.Time, lineCount int) {
end := time.Now()
// Sleep to ensure backend stores logs.
time.Sleep(time.Second * 60)
instanceId := awsservice.GetInstanceId()
err := awsservice.ValidateLogs(
instanceId,
instanceId,
&start,
&end,
// *2 because 2 lines per loop
awsservice.AssertLogsCount(lineCount),
awsservice.AssertNoDuplicateLogs(),
)
assert.NoError(t, err)

}

func writeSleepRestart(t *testing.T, f *os.File, configPath string, linesPerLoop int, doRestart bool) {
if doRestart {
common.StartAgent(configPath, true, false)
}
// Sleep to ensure agent detects file before it is written.
time.Sleep(sleepForFlush)
writeLogLines(t, f, linesPerLoop)
time.Sleep(sleepForFlush)
if doRestart {
common.StopAgent()
}
c, _ := filepath.Glob(logFilePath + "*")
assert.Equal(t, 1, len(c))
}

// TestAutoRemovalStopAgent configures agent to monitor a file with auto removal on.
// Then it restarts the agent.
// Verify the file is NOT removed.
Expand Down Expand Up @@ -196,7 +183,6 @@ func TestAutoRemovalFileRotation(t *testing.T) {
func TestRotatingLogsDoesNotSkipLines(t *testing.T) {
cfgFilePath := "resources/config_log_rotated.json"

instanceId := awsservice.GetInstanceId()
log.Printf("Found instance id %s", instanceId)
logGroup := instanceId
logStream := instanceId + "Rotated"
Expand Down Expand Up @@ -250,6 +236,42 @@ func TestRotatingLogsDoesNotSkipLines(t *testing.T) {
assert.NoError(t, err)
}

func TestLogGroupClass(t *testing.T) {
logFile, err := os.Create(logFilePath)
agentRuntime := 20 * time.Second // default flush interval is 5 seconds
if err != nil {
t.Fatalf("Error occurred creating log file for writing: %v", err)
}
defer logFile.Close()
defer os.Remove(logFilePath)

for _, param := range cloudWatchLogGroupClassTestParameters {
t.Run(param.testName, func(t *testing.T) {
defer awsservice.DeleteLogGroupAndStream(param.logGroupName, instanceId)
common.DeleteFile(common.AgentLogFile)
common.TouchFile(common.AgentLogFile)

common.CopyFile(param.configPath, configOutputPath)

err := common.StartAgent(configOutputPath, true, false)
assert.Nil(t, err)
// ensure that there is enough time from the "start" time and the first log line,
time.Sleep(agentRuntime)
writeLogLines(t, logFile, 100)
time.Sleep(agentRuntime)
common.StopAgent()

agentLog, err := os.ReadFile(common.AgentLogFile)
if err != nil {
return
}
t.Logf("Agent logs %s", string(agentLog))

assert.True(t, awsservice.IsLogGroupExists(param.logGroupName))
})
}
}

func writeLogLines(t *testing.T, f *os.File, iterations int) {
log.Printf("Writing %d lines to %s", iterations*len(logLineIds), f.Name())

Expand All @@ -266,3 +288,43 @@ func writeLogLines(t *testing.T, f *os.File, iterations int) {
time.Sleep(1 * time.Millisecond)
}
}

func writeSleepRestart(t *testing.T, f *os.File, configPath string, linesPerLoop int, doRestart bool) {
if doRestart {
common.StartAgent(configPath, true, false)
}
// Sleep to ensure agent detects file before it is written.
time.Sleep(sleepForFlush)
writeLogLines(t, f, linesPerLoop)
time.Sleep(sleepForFlush)
if doRestart {
common.StopAgent()
}
c, _ := filepath.Glob(logFilePath + "*")
assert.Equal(t, 1, len(c))
}

func autoRemovalTestCleanup() {
awsservice.DeleteLogGroupAndStream(instanceId, instanceId)
paths, _ := filepath.Glob(logFilePath + "*")
for _, p := range paths {
_ = os.Remove(p)
}
}

// checkData queries CWL and verifies the number of log lines.
func checkData(t *testing.T, start time.Time, lineCount int) {
end := time.Now()
// Sleep to ensure backend stores logs.
time.Sleep(time.Second * 60)
err := awsservice.ValidateLogs(
instanceId,
instanceId,
&start,
&end,
// *2 because 2 lines per loop
awsservice.AssertLogsCount(lineCount),
awsservice.AssertNoDuplicateLogs(),
)
assert.NoError(t, err)
}
21 changes: 21 additions & 0 deletions test/cloudwatchlogs/resources/config_log_infrequent_access.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"agent": {
"run_as_user": "root",
"debug": true
},
"logs": {
"logs_collected": {
"files": {
"collect_list": [
{
"file_path": "/tmp/cwagent_log_test.log",
"log_group_name": "{instance_id}-infrequent_access",
"log_stream_name": "{instance_id}",
"timezone": "UTC",
"log_group_class": "INFREQUENT_ACCESS"
}
]
}
}
}
}
21 changes: 21 additions & 0 deletions test/cloudwatchlogs/resources/config_log_standard_access.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"agent": {
"run_as_user": "root",
"debug": true
},
"logs": {
"logs_collected": {
"files": {
"collect_list": [
{
"file_path": "/tmp/cwagent_log_test.log",
"log_group_name": "{instance_id}-standard",
"log_stream_name": "{instance_id}",
"timezone": "UTC",
"log_group_class": "STANDARD"
}
]
}
}
}
}
Loading