Skip to content

Commit

Permalink
Default timestamp_format to be compatible with zero padding and non z…
Browse files Browse the repository at this point in the history
…ero padding options for month and day (#870)
  • Loading branch information
ymtaye authored Oct 16, 2023
1 parent ef67cac commit 5deb8c9
Show file tree
Hide file tree
Showing 12 changed files with 365 additions and 47 deletions.
4 changes: 2 additions & 2 deletions plugins/inputs/logfile/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ The plugin expects messages in one of the
log_group_name = "logfile.log"
log_stream_name = "<log_stream_name>"
timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$"
timestamp_layout = "02 Jan 2006 15:04:05"
timestamp_layout = ["_2 Jan 2006 15:04:05"]
timezone = "UTC"
multi_line_start_pattern = "{timestamp_regex}"
## Read file from beginning.
Expand All @@ -63,7 +63,7 @@ The plugin expects messages in one of the
log_group_name = "varlog"
log_stream_name = "<log_stream_name>"
timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$"
timestamp_layout = "02 Jan 2006 15:04:05"
timestamp_layout = ["_2 Jan 2006 15:04:05"]
timezone = "UTC"
multi_line_start_pattern = "{timestamp_regex}"
## Read file from beginning.
Expand Down
11 changes: 9 additions & 2 deletions plugins/inputs/logfile/fileconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ type FileConfig struct {
//The regex of the timestampFromLogLine presents in the log entry
TimestampRegex string `toml:"timestamp_regex"`
//The timestampFromLogLine layout used in GoLang to parse the timestampFromLogLine.
TimestampLayout string `toml:"timestamp_layout"`
TimestampLayout []string `toml:"timestamp_layout"`
//The time zone used to parse the timestampFromLogLine in the log entry.
Timezone string `toml:"timezone"`

Expand Down Expand Up @@ -179,7 +179,14 @@ func (config *FileConfig) timestampFromLogLine(logValue string) time.Time {
replacement := fmt.Sprintf(".%s", fracSecond[:3])
timestampContent = fmt.Sprintf("%s%s%s", timestampContent[:start], replacement, timestampContent[end:])
}
timestamp, err := time.ParseInLocation(config.TimestampLayout, timestampContent, config.TimezoneLoc)
var err error
var timestamp time.Time
for _, timestampLayout := range config.TimestampLayout {
timestamp, err = time.ParseInLocation(timestampLayout, timestampContent, config.TimezoneLoc)
if err == nil {
break
}
}
if err != nil {
log.Printf("E! Error parsing timestampFromLogLine: %s", err)
return time.Time{}
Expand Down
51 changes: 45 additions & 6 deletions plugins/inputs/logfile/fileconfig_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ func TestFileConfigInit(t *testing.T) {
FilePath: "/tmp/logfile.log",
LogGroupName: "logfile.log",
TimestampRegex: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})",
TimestampLayout: "02 Jan 2006 15:04:05",
TimestampLayout: []string{"02 Jan 2006 15:04:05"},
Timezone: "UTC",
MultiLineStartPattern: "{timestamp_regex}",
}
Expand Down Expand Up @@ -47,7 +47,7 @@ func TestFileConfigInitFailureCase(t *testing.T) {
FilePath: "/tmp/logfile.log",
LogGroupName: "logfile.log",
TimestampRegex: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}+)",
TimestampLayout: "02 Jan 2006 15:04:05",
TimestampLayout: []string{"02 Jan 2006 15:04:05"},
Timezone: "UTC",
MultiLineStartPattern: "{timestamp_regex}",
}
Expand All @@ -60,7 +60,7 @@ func TestFileConfigInitFailureCase(t *testing.T) {
FilePath: "/tmp/logfile.log",
LogGroupName: "logfile.log",
TimestampRegex: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})",
TimestampLayout: "02 Jan 2006 15:04:05",
TimestampLayout: []string{"02 Jan 2006 15:04:05"},
Timezone: "UTC",
MultiLineStartPattern: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}+)",
}
Expand All @@ -82,7 +82,7 @@ func TestLogGroupName(t *testing.T) {

func TestTimestampParser(t *testing.T) {
timestampRegex := "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})"
timestampLayout := "02 Jan 2006 15:04:05"
timestampLayout := []string{"02 Jan 2006 15:04:05"}
timezone := "UTC"
timezoneLoc := time.UTC
timestampRegexP, err := regexp.Compile(timestampRegex)
Expand Down Expand Up @@ -110,7 +110,7 @@ func TestTimestampParser(t *testing.T) {

func TestTimestampParserWithPadding(t *testing.T) {
timestampRegex := "(\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})"
timestampLayout := "1 2 15:04:05"
timestampLayout := []string{"1 2 15:04:05"}
timezone := "UTC"
timezoneLoc := time.UTC
timestampRegexP, err := regexp.Compile(timestampRegex)
Expand All @@ -133,9 +133,48 @@ func TestTimestampParserWithPadding(t *testing.T) {
assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute()))
}

func TestTimestampParserDefault(t *testing.T) {
// Check when timestamp_format is "%b %d %H:%M:%S"
// %d and %-d are both treated as s{0,1}\\d{1,2}
timestampRegex := "(\\w{3} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})"
timestampLayout := []string{"test", "Jan 2 15:04:05"}
timezone := "UTC"
timezoneLoc := time.UTC
timestampRegexP, err := regexp.Compile(timestampRegex)
require.NoError(t, err, fmt.Sprintf("Failed to compile regex %s", timestampRegex))
fileConfig := &FileConfig{
TimestampRegex: timestampRegex,
TimestampRegexP: timestampRegexP,
TimestampLayout: timestampLayout,
Timezone: timezone,
TimezoneLoc: timezoneLoc}

// make sure layout is compatible for "Sep 9", "Sep 9" , "Sep 09", "Sep 09" options
logEntry := fmt.Sprintf("Sep 9 02:00:43 ip-10-4-213-132 \n")
timestamp := fileConfig.timestampFromLogLine(logEntry)
assert.Equal(t, 02, timestamp.Hour())
assert.Equal(t, 00, timestamp.Minute())

logEntry = fmt.Sprintf("Sep 9 02:00:43 ip-10-4-213-132 \n")
timestamp = fileConfig.timestampFromLogLine(logEntry)
assert.Equal(t, 02, timestamp.Hour())
assert.Equal(t, 00, timestamp.Minute())

logEntry = fmt.Sprintf("Sep 09 02:00:43 ip-10-4-213-132 \n")
timestamp = fileConfig.timestampFromLogLine(logEntry)
assert.Equal(t, 02, timestamp.Hour())
assert.Equal(t, 00, timestamp.Minute())

logEntry = fmt.Sprintf("Sep 09 02:00:43 ip-10-4-213-132 \n")
timestamp = fileConfig.timestampFromLogLine(logEntry)
assert.Equal(t, 02, timestamp.Hour())
assert.Equal(t, 00, timestamp.Minute())

}

func TestTimestampParserWithFracSeconds(t *testing.T) {
timestampRegex := "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2},(\\d{1,9}) \\w{3})"
timestampLayout := "02 Jan 2006 15:04:05,.000 MST"
timestampLayout := []string{"02 Jan 2006 15:04:05,.000 MST"}
timezone := "UTC"
timezoneLoc := time.UTC
timestampRegexP, err := regexp.Compile(timestampRegex)
Expand Down
2 changes: 1 addition & 1 deletion plugins/inputs/logfile/logfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ const sampleConfig = `
log_stream_name = "<log_stream_name>"
publish_multi_logs = false
timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$"
timestamp_layout = "02 Jan 2006 15:04:05"
timestamp_layout = ["_2 Jan 2006 15:04:05"]
timezone = "UTC"
multi_line_start_pattern = "{timestamp_regex}"
## Read file from beginning.
Expand Down
2 changes: 1 addition & 1 deletion plugins/inputs/logfile/logfile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ append line`
FilePath: tmpfile.Name(),
FromBeginning: true,
TimestampRegex: "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\s{0,1}\\d{1,2})",
TimestampLayout: "15:04:05 06 Jan 2",
TimestampLayout: []string{"15:04:05 06 Jan 2"},
MultiLineStartPattern: "{timestamp_regex}",
Timezone: time.UTC.String(),
}}
Expand Down
13 changes: 11 additions & 2 deletions plugins/outputs/cloudwatchlogs/pusher.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ import (
)

const (
reqSizeLimit = 1024 * 1024
reqEventsLimit = 10000
reqSizeLimit = 1024 * 1024
reqEventsLimit = 10000
warnOldTimeStamp = 1 * 24 * time.Hour
)

var (
Expand Down Expand Up @@ -51,6 +52,7 @@ type pusher struct {
flushTimer *time.Timer
sequenceToken *string
lastValidTime int64
lastUpdateTime time.Time
needSort bool
stop <-chan struct{}
lastSentTime time.Time
Expand Down Expand Up @@ -413,12 +415,19 @@ func (p *pusher) convertEvent(e logs.LogEvent) *cloudwatchlogs.InputLogEvent {
// a valid timestamp and use the last valid timestamp for new entries that does
// not have a timestamp.
t = p.lastValidTime
if !p.lastUpdateTime.IsZero() {
// Check when timestamp has an interval of 5 days.
if time.Since(p.lastUpdateTime) > warnOldTimeStamp {
p.Log.Warnf("Unable to parse timestamp, using last valid timestamp found in the logs %v: which is at least older than 1 day for log group %v: ", p.lastValidTime, p.Group)
}
}
} else {
t = time.Now().UnixNano() / 1000000
}
} else {
t = e.Time().UnixNano() / 1000000
p.lastValidTime = t
p.lastUpdateTime = time.Now()
}
return &cloudwatchlogs.InputLogEvent{
Message: &message,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
multi_line_start_pattern = "{timestamp_regex}"
pipe = false
retention_in_days = -1
timestamp_layout = "02 Jan 2006 15:04:05"
timestamp_layout = ["_2 Jan 2006 15:04:05"]
timestamp_regex = "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})"
timezone = "UTC"

Expand Down
4 changes: 2 additions & 2 deletions translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
log_group_name = "amazon-cloudwatch-agent.log"
pipe = false
retention_in_days = -1
timestamp_layout = "15:04:05 06 Jan 02"
timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} d{2})"
timestamp_layout = ["15:04:05 06 Jan _2"]
timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} s{0,1} d{1,2})"

[outputs]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
log_group_name = "amazon-cloudwatch-agent.log"
pipe = false
retention_in_days = -1
timestamp_layout = "15:04:05 06 Jan 02"
timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} d{2})"
timestamp_layout = ["15:04:05 06 Jan _2"]
timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} s{0,1} d{1,2})"

[outputs]

Expand Down
Loading

0 comments on commit 5deb8c9

Please sign in to comment.