diff --git a/CHANGELOG.md b/CHANGELOG.md index 1874189a34a..348679246aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Here is an overview of all new **experimental** features: ### Improvements +- **GitHub Scaler**: Add support to use ETag for conditional requests against the Github API ([#6503](https://github.com/kedacore/keda/issues/6503)) - **IBMMQ Scaler**: Handling StatusNotFound in IBMMQ scaler ([#6472](https://github.com/kedacore/keda/pull/6472)) ### Fixes diff --git a/pkg/scalers/github_runner_scaler.go b/pkg/scalers/github_runner_scaler.go index 026c363709e..5ad18f89b31 100644 --- a/pkg/scalers/github_runner_scaler.go +++ b/pkg/scalers/github_runner_scaler.go @@ -30,10 +30,14 @@ const ( var reservedLabels = []string{"self-hosted", "linux", "x64"} type githubRunnerScaler struct { - metricType v2.MetricTargetType - metadata *githubRunnerMetadata - httpClient *http.Client - logger logr.Logger + metricType v2.MetricTargetType + metadata *githubRunnerMetadata + httpClient *http.Client + logger logr.Logger + etags map[string]string + previousRepos []string + previousWfrs map[string]*WorkflowRuns + previousJobs map[string][]Job } type githubRunnerMetadata struct { @@ -44,6 +48,7 @@ type githubRunnerMetadata struct { repos []string labels []string noDefaultLabels bool + enableEtags bool targetWorkflowQueueLength int64 triggerIndex int applicationID *int64 @@ -350,11 +355,20 @@ func NewGitHubRunnerScaler(config *scalersconfig.ScalerConfig) (Scaler, error) { httpClient = &http.Client{Transport: hc} } + etags := make(map[string]string) + previousRepos := []string{} + previousJobs := map[string][]Job{} + previousWfrs := map[string]*WorkflowRuns{} + return &githubRunnerScaler{ - metricType: metricType, - metadata: meta, - httpClient: httpClient, - logger: InitializeLogger(config, "github_runner_scaler"), + metricType: metricType, + metadata: meta, + httpClient: httpClient, + logger: InitializeLogger(config, "github_runner_scaler"), + etags: etags, + previousRepos: previousRepos, + previousJobs: previousJobs, + previousWfrs: previousWfrs, }, nil } @@ -432,6 +446,12 @@ func parseGitHubRunnerMetadata(config *scalersconfig.ScalerConfig) (*githubRunne meta.noDefaultLabels = false } + if val, err := getBoolValueFromMetaOrEnv("enableEtags", config.TriggerMetadata, config.ResolvedEnv); err == nil { + meta.enableEtags = val + } else { + meta.enableEtags = false + } + if val, err := getValueFromMetaOrEnv("repos", config.TriggerMetadata, config.ResolvedEnv); err == nil && val != "" { meta.repos = strings.Split(val, ",") } @@ -521,10 +541,17 @@ func (s *githubRunnerScaler) getRepositories(ctx context.Context) ([]string, err return nil, fmt.Errorf("runnerScope %s not supported", s.metadata.runnerScope) } - body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient) + body, statusCode, err := s.getGithubRequest(ctx, url, s.metadata, s.httpClient) if err != nil { return nil, err } + if statusCode == 304 { + if s.previousRepos != nil { + return s.previousRepos, nil + } else { + return nil, fmt.Errorf("request for repositories returned status: %d %s but previous repositories is not set", statusCode, http.StatusText(statusCode)) + } + } var repos []Repo @@ -545,10 +572,12 @@ func (s *githubRunnerScaler) getRepositories(ctx context.Context) ([]string, err page++ } + s.previousRepos = repoList + return repoList, nil } -func getGithubRequest(ctx context.Context, url string, metadata *githubRunnerMetadata, httpClient *http.Client) ([]byte, int, error) { +func (s *githubRunnerScaler) getGithubRequest(ctx context.Context, url string, metadata *githubRunnerMetadata, httpClient *http.Client) ([]byte, int, error) { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return []byte{}, -1, err @@ -561,6 +590,12 @@ func getGithubRequest(ctx context.Context, url string, metadata *githubRunnerMet req.Header.Set("Authorization", "Bearer "+*metadata.personalAccessToken) } + if s.metadata.enableEtags { + if etag, found := s.etags[url]; found { + req.Header.Set("If-None-Match", etag) + } + } + r, err := httpClient.Do(req) if err != nil { return []byte{}, -1, err @@ -573,6 +608,12 @@ func getGithubRequest(ctx context.Context, url string, metadata *githubRunnerMet _ = r.Body.Close() if r.StatusCode != 200 { + + if r.StatusCode == 304 { + s.logger.V(1).Info(fmt.Sprintf("The github rest api for the url: %s returned status %d %s", url, r.StatusCode, http.StatusText(r.StatusCode))) + return []byte{}, r.StatusCode, nil + } + if r.Header.Get("X-RateLimit-Remaining") != "" { githubAPIRemaining, _ := strconv.Atoi(r.Header.Get("X-RateLimit-Remaining")) @@ -585,6 +626,12 @@ func getGithubRequest(ctx context.Context, url string, metadata *githubRunnerMet return []byte{}, r.StatusCode, fmt.Errorf("the GitHub REST API returned error. url: %s status: %d response: %s", url, r.StatusCode, string(b)) } + if s.metadata.enableEtags { + if etag := r.Header.Get("ETag"); etag != "" { + s.etags[url] = etag + } + } + return b, r.StatusCode, nil } @@ -603,10 +650,17 @@ func stripDeadRuns(allWfrs []WorkflowRuns) []WorkflowRun { // getWorkflowRunJobs returns a list of jobs for a given workflow run func (s *githubRunnerScaler) getWorkflowRunJobs(ctx context.Context, workflowRunID int64, repoName string) ([]Job, error) { url := fmt.Sprintf("%s/repos/%s/%s/actions/runs/%d/jobs", s.metadata.githubAPIURL, s.metadata.owner, repoName, workflowRunID) - body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient) + body, statusCode, err := s.getGithubRequest(ctx, url, s.metadata, s.httpClient) if err != nil { return nil, err } + if statusCode == 304 { + if s.previousJobs[repoName] != nil { + return s.previousJobs[repoName], nil + } else { + return nil, fmt.Errorf("request for jobs returned status: %d %s but previous jobs is not set", statusCode, http.StatusText(statusCode)) + } + } var jobs Jobs err = json.Unmarshal(body, &jobs) @@ -614,18 +668,27 @@ func (s *githubRunnerScaler) getWorkflowRunJobs(ctx context.Context, workflowRun return nil, err } + s.previousJobs[repoName] = jobs.Jobs + return jobs.Jobs, nil } // getWorkflowRuns returns a list of workflow runs for a given repository func (s *githubRunnerScaler) getWorkflowRuns(ctx context.Context, repoName string) (*WorkflowRuns, error) { url := fmt.Sprintf("%s/repos/%s/%s/actions/runs", s.metadata.githubAPIURL, s.metadata.owner, repoName) - body, statusCode, err := getGithubRequest(ctx, url, s.metadata, s.httpClient) + body, statusCode, err := s.getGithubRequest(ctx, url, s.metadata, s.httpClient) if err != nil && statusCode == 404 { return nil, nil } else if err != nil { return nil, err } + if statusCode == 304 { + if s.previousWfrs[repoName] != nil { + return s.previousWfrs[repoName], nil + } else { + return nil, fmt.Errorf("request for workflow runs returned status: %d %s but previous workflow runs is not set", statusCode, http.StatusText(statusCode)) + } + } var wfrs WorkflowRuns err = json.Unmarshal(body, &wfrs) @@ -633,6 +696,8 @@ func (s *githubRunnerScaler) getWorkflowRuns(ctx context.Context, repoName strin return nil, err } + s.previousWfrs[repoName] = &wfrs + return &wfrs, nil }