Skip to content

Commit

Permalink
Add option to configure HTTP request timeout in milliseconds
Browse files Browse the repository at this point in the history 
Keep (but also deprecate) old option `request_timeout_in_seconds` to preserve backward compatibility.
  • Loading branch information
@pondzix
pondzix committed Jan 30, 2025
1 parent 6d51561 commit 38c7ef9
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 29 deletions.
3 changes: 3 additions & 0 deletions assets/docs/configuration/targets/http-full-example.hcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ target {
# Request timeout in seconds (default: 5)
request_timeout_in_seconds = 2

# Request timeout in milliseconds (default: 5000)
request_timeout_in_millis = 2000

# Content type for POST request (default: "application/json")
content_type = "text/html"

Expand Down
4 changes: 3 additions & 1 deletion config/component_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ func TestCreateTargetComponentHCL(t *testing.T) {
RequestMaxMessages: 20,
RequestByteLimit: 1048576,
MessageByteLimit: 1048576,
RequestTimeoutInSeconds: 5,
RequestTimeoutInSeconds: 0,
RequestTimeoutInMillis: 0,
ContentType: "application/json",
Headers: "",
BasicAuthUsername: "",
Expand All @@ -104,6 +105,7 @@ func TestCreateTargetComponentHCL(t *testing.T) {
RequestByteLimit: 1000000,
MessageByteLimit: 1000000,
RequestTimeoutInSeconds: 2,
RequestTimeoutInMillis: 2000,
ContentType: "text/html",
Headers: "{\"Accept-Language\":\"en-US\"}",
BasicAuthUsername: "myUsername",
Expand Down
31 changes: 26 additions & 5 deletions pkg/target/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
type HTTPTargetConfig struct {
HTTPURL string `hcl:"url"`
RequestTimeoutInSeconds int `hcl:"request_timeout_in_seconds,optional"`
RequestTimeoutInMillis int `hcl:"request_timeout_in_millis,optional"`
ContentType string `hcl:"content_type,optional"`
Headers string `hcl:"headers,optional"`
BasicAuthUsername string `hcl:"basic_auth_username,optional"`
Expand Down Expand Up @@ -145,7 +146,7 @@ func addHeadersToRequest(request *http.Request, headers map[string]string, dynam
// newHTTPTarget creates a client for writing events to HTTP
func newHTTPTarget(
httpURL string,
requestTimeout int,
requestTimeoutMillis int,
requestMaxMessages int,
requestByteLimit int,
messageByteLimit int,
Expand Down Expand Up @@ -187,7 +188,7 @@ func newHTTPTarget(
}

client := createHTTPClient(oAuth2ClientID, oAuth2ClientSecret, oAuth2TokenURL, oAuth2RefreshToken, transport)
client.Timeout = time.Duration(requestTimeout) * time.Second
client.Timeout = time.Duration(requestTimeoutMillis) * time.Millisecond

approxTmplSize, requestTemplate, err := loadRequestTemplate(templateFile)
if err != nil {
Expand Down Expand Up @@ -276,9 +277,30 @@ func createHTTPClient(oAuth2ClientID string, oAuth2ClientSecret string, oAuth2To

// HTTPTargetConfigFunction creates HTTPTarget from HTTPTargetConfig
func HTTPTargetConfigFunction(c *HTTPTargetConfig) (*HTTPTarget, error) {
var requestTimeoutInMillis int

if c.RequestTimeoutInMillis != 0 && c.RequestTimeoutInSeconds == 0 {
requestTimeoutInMillis = c.RequestTimeoutInMillis
}

if c.RequestTimeoutInMillis != 0 && c.RequestTimeoutInSeconds != 0 {
requestTimeoutInMillis = c.RequestTimeoutInMillis
log.Warn("Both 'request_timeout' and 'request_timeout_in_seconds' options are set. In this case 'request_timeout' takes precendence and 'request_timeout_in_seconds' is ignored. Using 'request_timeout_in_seconds' is deprecated, and will be removed in the next major version. Use 'request_timeout' only")
}

if c.RequestTimeoutInMillis == 0 && c.RequestTimeoutInSeconds != 0 {
requestTimeoutInMillis = c.RequestTimeoutInSeconds * 1000
log.Warn("For the HTTP target, 'request_timeout_in_seconds' is deprecated, and will be removed in the next major version. Use 'request_timeout' instead")
}

if c.RequestTimeoutInMillis == 0 && c.RequestTimeoutInSeconds == 0 {
requestTimeoutInMillis = 5000
log.Warn("Neither 'request_timeout' nor 'request_timeout_in_seconds' are set. The previous default is preserved, but strongly advise manual configuration of 'request_timeout'")
}

return newHTTPTarget(
c.HTTPURL,
c.RequestTimeoutInSeconds,
requestTimeoutInMillis,
c.RequestMaxMessages,
c.RequestByteLimit,
c.MessageByteLimit,
Expand Down Expand Up @@ -321,8 +343,7 @@ func (f HTTPTargetAdapter) ProvideDefault() (interface{}, error) {
MessageByteLimit: 1048576,
EnableTLS: false,

RequestTimeoutInSeconds: 5,
ContentType: "application/json",
ContentType: "application/json",
ResponseRules: &ResponseRules{
Invalid: []Rule{},
SetupError: []Rule{},
Expand Down
2 changes: 1 addition & 1 deletion pkg/target/http_oauth2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func runTest(t *testing.T, inputClientID string, inputClientSecret string, input
}

func oauth2Target(t *testing.T, targetURL string, inputClientID string, inputClientSecret string, inputRefreshToken string, tokenServerURL string) *HTTPTarget {
target, err := newHTTPTarget(targetURL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, inputClientID, inputClientSecret, inputRefreshToken, tokenServerURL, "", defaultResponseRules(), false)
target, err := newHTTPTarget(targetURL, 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, inputClientID, inputClientSecret, inputRefreshToken, tokenServerURL, "", defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down
84 changes: 62 additions & 22 deletions pkg/target/http_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,12 @@ func TestHTTP_RetrieveHeaders(t *testing.T) {
for _, tt := range testCases {
t.Run(tt.Name, func(t *testing.T) {
testTargetConfig := &HTTPTargetConfig{
HTTPURL: "http://test",
MessageByteLimit: 1048576,
RequestByteLimit: 1048576,
RequestTimeoutInSeconds: 5,
ContentType: "application/json",
DynamicHeaders: tt.Dynamic,
HTTPURL: "http://test",
MessageByteLimit: 1048576,
RequestByteLimit: 1048576,
RequestTimeoutInMillis: 5000,
ContentType: "application/json",
DynamicHeaders: tt.Dynamic,
}
testTarget, err := HTTPTargetConfigFunction(testTargetConfig)
if err != nil {
Expand All @@ -184,6 +184,46 @@ func TestHTTP_RetrieveHeaders(t *testing.T) {
}
}

func TestHTTP_RequestTimeoutsConfig(t *testing.T) {
testCases := []struct {
Name string
Config *HTTPTargetConfig
ExpectedCientTimeout time.Duration
}{
{
Name: "Nothing set",
Config: &HTTPTargetConfig{},
ExpectedCientTimeout: time.Duration(5) * time.Second,
},
{
Name: "In seconds only",
Config: &HTTPTargetConfig{RequestTimeoutInSeconds: 10},
ExpectedCientTimeout: time.Duration(10) * time.Second,
},
{
Name: "In milliseconds only",
Config: &HTTPTargetConfig{RequestTimeoutInMillis: 2500},
ExpectedCientTimeout: time.Duration(2500) * time.Millisecond,
},
{
Name: "Seconds and millis are set",
Config: &HTTPTargetConfig{RequestTimeoutInSeconds: 10, RequestTimeoutInMillis: 2500},
ExpectedCientTimeout: time.Duration(2500) * time.Millisecond,
},
}
for _, tt := range testCases {
t.Run(tt.Name, func(t *testing.T) {
assert := assert.New(t)
tt.Config.HTTPURL = "http://test"
tt.Config.RequestByteLimit = 1048576
tt.Config.MessageByteLimit = 1048576
target, _ := HTTPTargetConfigFunction(tt.Config)

assert.Equal(tt.ExpectedCientTimeout, target.client.Timeout)
})
}
}

func TestHTTP_AddHeadersToRequest(t *testing.T) {
assert := assert.New(t)

Expand Down Expand Up @@ -311,20 +351,20 @@ func TestHTTP_AddHeadersToRequest_WithDynamicHeaders(t *testing.T) {
func TestHTTP_NewHTTPTarget(t *testing.T) {
assert := assert.New(t)

httpTarget, err := newHTTPTarget("http://something", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
httpTarget, err := newHTTPTarget("http://something", 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)

assert.Nil(err)
assert.NotNil(httpTarget)

failedHTTPTarget, err1 := newHTTPTarget("something", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
failedHTTPTarget, err1 := newHTTPTarget("something", 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)

assert.NotNil(err1)
if err1 != nil {
assert.Equal("Invalid url for HTTP target: 'something'", err1.Error())
}
assert.Nil(failedHTTPTarget)

failedHTTPTarget2, err2 := newHTTPTarget("", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
failedHTTPTarget2, err2 := newHTTPTarget("", 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
assert.NotNil(err2)
if err2 != nil {
assert.Equal("Invalid url for HTTP target: ''", err2.Error())
Expand Down Expand Up @@ -353,7 +393,7 @@ func TestHTTP_Write_Simple(t *testing.T) {
server := createTestServerWithResponseCode(&results, &headers, tt.ResponseCode, "")
defer server.Close()

target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
target, err := newHTTPTarget(server.URL, 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -421,7 +461,7 @@ func TestHTTP_Write_Batched(t *testing.T) {
server := createTestServerWithResponseCode(&results, &headers, 200, "")
defer server.Close()

target, err := newHTTPTarget(server.URL, 5, tt.BatchSize, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
target, err := newHTTPTarget(server.URL, 5000, tt.BatchSize, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -480,7 +520,7 @@ func TestHTTP_Write_Concurrent(t *testing.T) {
server := createTestServer(&results)
defer server.Close()

target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
target, err := newHTTPTarget(server.URL, 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -524,7 +564,7 @@ func TestHTTP_Write_Failure(t *testing.T) {
server := createTestServer(&results)
defer server.Close()

target, err := newHTTPTarget("http://NonexistentEndpoint", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
target, err := newHTTPTarget("http://NonexistentEndpoint", 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -567,7 +607,7 @@ func TestHTTP_Write_InvalidResponseCode(t *testing.T) {
var headers http.Header
server := createTestServerWithResponseCode(&results, &headers, tt.ResponseCode, "")
defer server.Close()
target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
target, err := newHTTPTarget(server.URL, 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -602,7 +642,7 @@ func TestHTTP_Write_Oversized(t *testing.T) {
server := createTestServer(&results)
defer server.Close()

target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
target, err := newHTTPTarget(server.URL, 5000, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -645,7 +685,7 @@ func TestHTTP_Write_EnabledTemplating(t *testing.T) {
server := createTestServer(&results)
defer server.Close()

target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), defaultResponseRules(), false)
target, err := newHTTPTarget(server.URL, 5000, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), defaultResponseRules(), false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -705,7 +745,7 @@ func TestHTTP_Write_Invalid(t *testing.T) {
},
}

target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules, false)
target, err := newHTTPTarget(server.URL, 5000, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules, false)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -735,7 +775,7 @@ func TestHTTP_Write_Setup(t *testing.T) {
},
}

target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules, false)
target, err := newHTTPTarget(server.URL, 5000, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules, false)
if err != nil {
t.Fatal(err)
}
Expand All @@ -762,7 +802,7 @@ func TestHTTP_TimeOrientedHeadersEnabled(t *testing.T) {
server := createTestServerWithResponseCode(&results, &headers, 200, "ok")
defer server.Close()

target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), true)
target, err := newHTTPTarget(server.URL, 5000, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules(), true)
if err != nil {
t.Fatal(err)
}
Expand All @@ -789,7 +829,7 @@ func TestHTTP_Write_TLS(t *testing.T) {

// Test that https requests work with manually provided certs
target, err := newHTTPTarget("https://localhost:8999/hello",
5,
5000,
1,
1048576,
1048576,
Expand Down Expand Up @@ -831,7 +871,7 @@ func TestHTTP_Write_TLS(t *testing.T) {

// Test that https requests work for different endpoints when different certs are provided manually
target2, err2 := newHTTPTarget(ngrokAddress,
5,
5000,
1,
1048576,
1048576,
Expand Down Expand Up @@ -866,7 +906,7 @@ func TestHTTP_Write_TLS(t *testing.T) {

// Test that https requests work for different endpoints when different certs are provided manually
target3, err4 := newHTTPTarget(ngrokAddress,
5,
5000,
1,
1048576,
1048576,
Expand Down

0 comments on commit 38c7ef9

Please sign in to comment.