From a26f7e10412f7e53f9518c6387450d5ddbc92e45 Mon Sep 17 00:00:00 2001 From: David Cauchi <13139524+davidcauchi@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:49:41 +0200 Subject: [PATCH] [ship-3765]Adds wrapper for Loki Rest api to query logs (#1203) add Loki client for queries --- lib/.changeset/v1.50.10.md | 1 + lib/README.md | 30 ++++++ lib/client/loki.go | 184 +++++++++++++++++++++++++++++++++++++ lib/client/loki_test.go | 181 ++++++++++++++++++++++++++++++++++++ lib/logging/log.go | 3 + 5 files changed, 399 insertions(+) create mode 100644 lib/.changeset/v1.50.10.md create mode 100644 lib/client/loki.go create mode 100644 lib/client/loki_test.go diff --git a/lib/.changeset/v1.50.10.md b/lib/.changeset/v1.50.10.md new file mode 100644 index 000000000..9ee955d79 --- /dev/null +++ b/lib/.changeset/v1.50.10.md @@ -0,0 +1 @@ +- Added Loki client to query logs data + tests, see usage here - [README](../../README.md) \ No newline at end of file diff --git a/lib/README.md b/lib/README.md index ca26b360f..186d71dae 100644 --- a/lib/README.md +++ b/lib/README.md @@ -430,3 +430,33 @@ export RESTY_DEBUG=true ## Using AWS Secrets Manager Check the [docs](SECRETS.md) + +## Loki Client + +The `LokiClient` allows you to easily query Loki logs from your tests. It supports basic authentication, custom queries, and can be configured for (Resty) debug mode. + +### Debugging Resty and Loki Client + +```bash +export LOKI_CLIENT_LOG_LEVEL=info +export RESTY_DEBUG=true +``` + +### Example usage: + +```go +auth := LokiBasicAuth{ + Username: os.Getenv("LOKI_LOGIN"), + Password: os.Getenv("LOKI_PASSWORD"), +} + +queryParams := LokiQueryParams{ + Query: `{namespace="test"} |= "test"`, + StartTime: time.Now().AddDate(0, 0, -1), + EndTime: time.Now(), + Limit: 100, + } + +lokiClient := NewLokiClient("https://loki.api.url", "my-tenant", auth, queryParams) +logEntries, err := lokiClient.QueryLogs(context.Background()) +``` diff --git a/lib/client/loki.go b/lib/client/loki.go new file mode 100644 index 000000000..344dbf82f --- /dev/null +++ b/lib/client/loki.go @@ -0,0 +1,184 @@ +package client + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/go-resty/resty/v2" + + "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" +) + +// LokiAPIError is a custom error type for handling non-200 responses from the Loki API +type LokiAPIError struct { + StatusCode int + Message string +} + +// Implement the `Error` interface for LokiAPIError +func (e *LokiAPIError) Error() string { + return fmt.Sprintf("Loki API error: %s (status code: %d)", e.Message, e.StatusCode) +} + +// LokiBasicAuth holds the authentication details for Loki +type LokiBasicAuth struct { + Login string + Password string +} + +// LokiResponse represents the structure of the response from Loki +type LokiResponse struct { + Data struct { + Result []struct { + Stream map[string]string `json:"stream"` + Values [][]interface{} `json:"values"` + } `json:"result"` + } `json:"data"` +} + +// LokiLogEntry represents a single log entry with a timestamp and raw log message +type LokiLogEntry struct { + Timestamp string + Log string +} + +// LokiClient represents a client to interact with Loki for querying logs +type LokiClient struct { + BaseURL string + TenantID string + BasicAuth LokiBasicAuth + QueryParams LokiQueryParams + Logger logging.Logger + RestyClient *resty.Client +} + +// LokiQueryParams holds the parameters required for querying Loki +type LokiQueryParams struct { + Query string + StartTime time.Time + EndTime time.Time + Limit int +} + +// NewLokiClient creates a new Loki client with the given parameters, initializes a logger, and configures Resty with debug mode +func NewLokiClient(baseURL, tenantID string, auth LokiBasicAuth, queryParams LokiQueryParams) *LokiClient { + logging.Init() + + logger := logging.GetLogger(nil, "LOKI_CLIENT_LOG_LEVEL") + logger.Info(). + Str("BaseURL", baseURL). + Str("TenantID", tenantID). + Msg("Initializing Loki Client") + + // Set debug mode for Resty if RESTY_DEBUG is enabled + isDebug := os.Getenv("RESTY_DEBUG") == "true" + + restyClient := resty.New(). + SetDebug(isDebug) + + return &LokiClient{ + BaseURL: baseURL, + TenantID: tenantID, + BasicAuth: auth, + QueryParams: queryParams, + Logger: logger, + RestyClient: restyClient, + } +} + +// QueryLogs queries Loki logs based on the query parameters and returns the raw log entries +func (lc *LokiClient) QueryLogs(ctx context.Context) ([]LokiLogEntry, error) { + // Log request details + lc.Logger.Info(). + Str("Query", lc.QueryParams.Query). + Str("StartTime", lc.QueryParams.StartTime.Format(time.RFC3339Nano)). + Str("EndTime", lc.QueryParams.EndTime.Format(time.RFC3339Nano)). + Int("Limit", lc.QueryParams.Limit). + Msg("Making request to Loki API") + + // Start tracking request duration + start := time.Now() + + // Build query parameters + params := map[string]string{ + "query": lc.QueryParams.Query, + "start": lc.QueryParams.StartTime.Format(time.RFC3339Nano), + "end": lc.QueryParams.EndTime.Format(time.RFC3339Nano), + "limit": fmt.Sprintf("%d", lc.QueryParams.Limit), + } + + // Send request using the pre-configured Resty client + resp, err := lc.RestyClient.R(). + SetContext(ctx). + SetHeader("X-Scope-OrgID", lc.TenantID). + SetBasicAuth(lc.BasicAuth.Login, lc.BasicAuth.Password). + SetQueryParams(params). + Get(lc.BaseURL + "/loki/api/v1/query_range") + + // Track request duration + duration := time.Since(start) + + if err != nil { + lc.Logger.Error().Err(err).Dur("duration", duration).Msg("Error querying Loki") + return nil, err + } + + // Log non-200 responses + if resp.StatusCode() != 200 { + bodySnippet := string(resp.Body()) + if len(bodySnippet) > 200 { + bodySnippet = bodySnippet[:200] + "..." + } + lc.Logger.Error(). + Int("StatusCode", resp.StatusCode()). + Dur("duration", duration). + Str("ResponseBody", bodySnippet). + Msg("Loki API returned non-200 status") + return nil, &LokiAPIError{ + StatusCode: resp.StatusCode(), + Message: "unexpected status code from Loki API", + } + } + + // Log successful response + lc.Logger.Info(). + Int("StatusCode", resp.StatusCode()). + Dur("duration", duration). + Msg("Successfully queried Loki API") + + // Parse the response into the LokiResponse struct + var lokiResp LokiResponse + if err := json.Unmarshal(resp.Body(), &lokiResp); err != nil { + lc.Logger.Error().Err(err).Msg("Error decoding response from Loki") + return nil, err + } + + // Extract log entries from the response + logEntries := lc.extractRawLogEntries(lokiResp) + + // Log the number of entries retrieved + lc.Logger.Info().Int("LogEntries", len(logEntries)).Msg("Successfully retrieved logs from Loki") + + return logEntries, nil +} + +// extractRawLogEntries processes the LokiResponse and returns raw log entries +func (lc *LokiClient) extractRawLogEntries(lokiResp LokiResponse) []LokiLogEntry { + var logEntries []LokiLogEntry + + for _, result := range lokiResp.Data.Result { + for _, entry := range result.Values { + timestamp := entry[0].(string) + logLine := entry[1].(string) + logEntries = append(logEntries, LokiLogEntry{ + Timestamp: timestamp, + Log: logLine, + }) + } + } + + return logEntries +} diff --git a/lib/client/loki_test.go b/lib/client/loki_test.go new file mode 100644 index 000000000..9d2b77810 --- /dev/null +++ b/lib/client/loki_test.go @@ -0,0 +1,181 @@ +package client + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestLokiClient_QueryLogs tests the LokiClient's ability to query Loki logs +func TestLokiClient_SuccessfulQuery(t *testing.T) { + // Create a mock Loki server using httptest + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/loki/api/v1/query_range", r.URL.Path) + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{ + "data": { + "result": [ + { + "stream": {"namespace": "test"}, + "values": [ + ["1234567890", "Log message 1"], + ["1234567891", "Log message 2"] + ] + } + ] + } + }`)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + // Create a BasicAuth object for testing + auth := LokiBasicAuth{ + Login: "test-login", + Password: "test-password", + } + + // Set the query parameters + queryParams := LokiQueryParams{ + Query: `{namespace="test"}`, + StartTime: time.Now().Add(-1 * time.Hour), + EndTime: time.Now(), + Limit: 100, + } + + // Create the Loki client with the mock server URL + lokiClient := NewLokiClient(mockServer.URL, "test-tenant", auth, queryParams) + + // Query logs + logEntries, err := lokiClient.QueryLogs(context.Background()) + assert.NoError(t, err) + assert.Len(t, logEntries, 2) + + // Verify the content of the log entries + assert.Equal(t, "1234567890", logEntries[0].Timestamp) + assert.Equal(t, "Log message 1", logEntries[0].Log) + assert.Equal(t, "1234567891", logEntries[1].Timestamp) + assert.Equal(t, "Log message 2", logEntries[1].Log) +} + +func TestLokiClient_AuthenticationFailure(t *testing.T) { + // Create a mock Loki server + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/loki/api/v1/query_range", r.URL.Path) + w.WriteHeader(http.StatusUnauthorized) // Simulate authentication failure + })) + defer mockServer.Close() + + // Create a Loki client with incorrect credentials + auth := LokiBasicAuth{ + Login: "wrong-login", + Password: "wrong-password", + } + queryParams := LokiQueryParams{ + Query: `{namespace="test"}`, + StartTime: time.Now().Add(-1 * time.Hour), + EndTime: time.Now(), + Limit: 100, + } + lokiClient := NewLokiClient(mockServer.URL, "test-tenant", auth, queryParams) + + // Query logs and expect an error + logEntries, err := lokiClient.QueryLogs(context.Background()) + assert.Nil(t, logEntries) + assert.Error(t, err) + var lokiErr *LokiAPIError + if errors.As(err, &lokiErr) { + assert.Equal(t, http.StatusUnauthorized, lokiErr.StatusCode) + } else { + t.Fatalf("Expected LokiAPIError, got %v", err) + } +} + +func TestLokiClient_InternalServerError(t *testing.T) { + // Create a mock Loki server + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/loki/api/v1/query_range", r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) // Simulate server error + _, err := w.Write([]byte(`{"message": "internal server error"}`)) // Error message in the response body + assert.NoError(t, err) + })) + defer mockServer.Close() + + // Create a Loki client + auth := LokiBasicAuth{ + Login: "test-login", + Password: "test-password", + } + queryParams := LokiQueryParams{ + Query: `{namespace="test"}`, + StartTime: time.Now().Add(-1 * time.Hour), + EndTime: time.Now(), + Limit: 100, + } + lokiClient := NewLokiClient(mockServer.URL, "test-tenant", auth, queryParams) + + // Query logs and expect an error + logEntries, err := lokiClient.QueryLogs(context.Background()) + assert.Nil(t, logEntries) + assert.Error(t, err) + var lokiErr *LokiAPIError + if errors.As(err, &lokiErr) { + assert.Equal(t, http.StatusInternalServerError, lokiErr.StatusCode) + } else { + t.Fatalf("Expected LokiAPIError, got %v", err) + } +} + +func TestLokiClient_DebugMode(t *testing.T) { + // Set the RESTY_DEBUG environment variable + os.Setenv("RESTY_DEBUG", "true") + defer os.Unsetenv("RESTY_DEBUG") // Clean up after the test + + // Create a mock Loki server + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/loki/api/v1/query_range", r.URL.Path) + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{ + "data": { + "result": [ + { + "stream": {"namespace": "test"}, + "values": [ + ["1234567890", "Log message 1"], + ["1234567891", "Log message 2"] + ] + } + ] + } + }`)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + // Create a Loki client + auth := LokiBasicAuth{ + Login: "test-login", + Password: "test-password", + } + queryParams := LokiQueryParams{ + Query: `{namespace="test"}`, + StartTime: time.Now().Add(-1 * time.Hour), + EndTime: time.Now(), + Limit: 100, + } + lokiClient := NewLokiClient(mockServer.URL, "test-tenant", auth, queryParams) + + // Query logs + logEntries, err := lokiClient.QueryLogs(context.Background()) + assert.NoError(t, err) + assert.Len(t, logEntries, 2) + + // Check if debug mode was enabled + assert.True(t, lokiClient.RestyClient.Debug) +} diff --git a/lib/logging/log.go b/lib/logging/log.go index f067a6053..2ad3cb670 100644 --- a/lib/logging/log.go +++ b/lib/logging/log.go @@ -17,6 +17,9 @@ import ( const afterTestEndedMsg = "LOG AFTER TEST ENDED" +// Logger is an alias for zerolog.Logger, exposed through the logging package +type Logger = zerolog.Logger + // CustomT wraps testing.T for two purposes: // 1. it implements Write to override the default logger // 2. it implements Printf to implement the testcontainers-go/Logging interface