diff --git a/README.md b/README.md index af0e2c0..fd01c01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# GitHub Actions Aggregator Service +# GitHub Actions Aggregator -A Go-based service to aggregate and analyze data from GitHub Actions workflows across multiple repositories. This application provides insights into workflow runs, success rates, failure rates, and other statistics over customizable time ranges. +GitHub Actions Aggregator is a Go-based service designed to collect, aggregate, and analyze data from GitHub Actions workflows across multiple repositories. This application provides valuable insights into workflow runs, success rates, failure rates, and other statistics over customizable time ranges. ## Table of Contents @@ -17,7 +17,7 @@ A Go-based service to aggregate and analyze data from GitHub Actions workflows a ## Features -- **OAuth 2.0 Authentication**: Securely authenticate users via GitHub OAuth. +- **OAuth 2.0 Authentication**: Secure user authentication via GitHub OAuth. - **Data Collection**: - **Webhooks**: Receive real-time updates on workflow events. - **Polling**: Periodically poll GitHub API to ensure data completeness. @@ -29,11 +29,10 @@ A Go-based service to aggregate and analyze data from GitHub Actions workflows a ## Prerequisites -- **Go**: Version 1.18 or higher. -- **GitHub Account**: For OAuth authentication and API access. -- **PostgreSQL**: For storing data. -- **Redis** (optional): For caching (if implemented). -- **Docker** (optional): For containerization and deployment. +- Go (version 1.18 or higher) +- PostgreSQL +- GitHub Account (for OAuth authentication and API access) +- Docker (optional, for containerization) ## Installation @@ -44,16 +43,15 @@ A Go-based service to aggregate and analyze data from GitHub Actions workflows a cd github-actions-aggregator ``` -2. **Install Dependencies** - - ```bash +2. Install dependencies: + ``` go mod download ``` -3. **Set Up Environment Variables** - +3. Set up environment variables: Create a `.env` file or export the required environment variables: + ```bash export GITHUB_CLIENT_ID="your_github_client_id" export GITHUB_CLIENT_SECRET="your_github_client_secret" @@ -63,11 +61,12 @@ A Go-based service to aggregate and analyze data from GitHub Actions workflows a export SERVER_PORT="8080" ``` + ## Configuration Configuration can be managed via a `config.yaml` file in the `configs/` directory or through environment variables. -**Example `config.yaml`:** +Example `config.yaml`: ```yaml server: @@ -83,126 +82,68 @@ github: webhook_secret: "your_webhook_secret" ``` -**Note:** Environment variables override values in the configuration file. +Note: Environment variables override values in the configuration file. ## Usage -### Running the Application - -1. **Run Database Migrations** - - ```bash - ./scripts/migrate.sh +1. Run database migrations: + ``` + ./scripts/migrate.sh up ``` -2. **Start the Application** - - ```bash +2. Start the application: + ``` go run cmd/server/main.go ``` -### Accessing the Application - -- **Login with GitHub**: Navigate to `http://localhost:8080/login` to authenticate via GitHub. -- **API Requests**: Use tools like `curl` or Postman to interact with the API endpoints. +3. Access the application: + - Login with GitHub: Navigate to `http://localhost:8080/login` + - API Requests: Use tools like `curl` or Postman to interact with the API endpoints ## API Endpoints -### Authentication - - `GET /login`: Redirects the user to GitHub for OAuth authentication. - `GET /callback`: Handles the OAuth callback from GitHub. - -### Workflow Statistics - - `GET /workflows/:id/stats`: Retrieves statistics for a specific workflow. +- `GET /repositories/:id/workflows`: Get all workflows for a repository +- `GET /workflows/:id/runs`: Get all runs for a workflow +- `GET /runs/:id`: Get a specific run +- `GET /jobs/:id`: Get a specific job +- `GET /jobs/:id/steps`: Get all steps for a job +- `GET /jobs/:id/stats`: Get stats for a job - **Query Parameters:** - - - `start_time` (optional): Start of the time range (ISO 8601 format). - - `end_time` (optional): End of the time range (ISO 8601 format). - - **Example Request:** - - Explicit time range: - - ```http - GET /workflows/123/stats?start_time=2023-09-01T00:00:00Z&end_time=2023-09-30T23:59:59Z - ``` - - Relative time range: - - ```http - GET /workflows/123/stats?start_time=24_hours&end_time=now - ``` - - **Example Response:** - - ```json - { - "workflow_id": 123, - "workflow_name": "CI Build and Test", - "total_runs": 200, - "success_count": 150, - "failure_count": 30, - "cancelled_count": 10, - "timed_out_count": 5, - "action_required_count": 5, - "success_rate": 75.0, - "failure_rate": 15.0, - "cancelled_rate": 5.0, - "timed_out_rate": 2.5, - "action_required_rate": 2.5, - "start_time": "2023-09-01T00:00:00Z", - "end_time": "2023-09-30T23:59:59Z" - } - ``` +For detailed information on request parameters and response formats, please refer to the API documentation. ## Authentication -### Setting Up OAuth with GitHub - -1. **Register a New OAuth Application** - - - Go to [GitHub Developer Settings](https://github.com/settings/developers). - - Click on **"New OAuth App"**. +1. Register a new OAuth application on GitHub: + - Go to GitHub Developer Settings + - Click on "New OAuth App" - Fill in the application details: - - **Application Name** - - **Homepage URL**: `http://localhost:8080` - - **Authorization Callback URL**: `http://localhost:8080/callback` - - Obtain your **Client ID** and **Client Secret**. - -2. **Configure Application Credentials** + - Application Name + - Homepage URL: `http://localhost:8080` + - Authorization Callback URL: `http://localhost:8080/callback` + - Obtain your Client ID and Client Secret +2. Configure application credentials: Set your `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` in your environment variables or `config.yaml`. -### Permissions and Scopes - -Ensure that your GitHub OAuth application has the necessary scopes: - -- `read:user` -- `repo` -- `workflow` +Ensure that your GitHub OAuth application has the necessary scopes: `read:user`, `repo`, and `workflow`. ## Testing -### Running Unit Tests - -```bash +Run unit tests: +``` go test ./tests/unit/... ``` -### Running Integration Tests - -```bash +Run integration tests: +``` go test ./tests/integration/... ``` -### Test Coverage - -You can generate a test coverage report using: - -```bash +Generate a test coverage report: +``` go test -coverprofile=coverage.out ./... go tool cover -html=coverage.out ``` @@ -241,11 +182,11 @@ Contributions are welcome! Please follow these steps: 6. **Create a Pull Request** - Go to the original repository and open a pull request. +For more details, please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file. ## License -This project is licensed under the [MIT License](LICENSE). +This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. --- diff --git a/cmd/server/main.go b/cmd/server/main.go index 6bc7d65..07caaf9 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -37,11 +37,16 @@ func main() { // Initialize GitHub client githubClient := github.NewClient(cfg.GitHub.AccessToken) - workerPool := worker.NewWorkerPool(database, cfg.WorkerPoolSize) - workerPool.Start() + // Initialize worker pool for polling + pollingWorkerPool := worker.NewWorkerPool(database, cfg.PollingWorkerPoolSize) + pollingWorkerPool.Start() + + // Initialize worker pool for webhooks + webhookWorkerPool := worker.NewWorkerPool(database, cfg.WebhookWorkerPoolSize) + webhookWorkerPool.Start() // Start the API server - go api.StartServer(cfg, database, githubClient, workerPool) + go api.StartServer(cfg, database, githubClient, webhookWorkerPool) // Set up graceful shutdown quit := make(chan os.Signal, 1) @@ -50,9 +55,9 @@ func main() { log.Println("Shutting down server...") - // Stop the worker pool - workerPool.Stop() - + // Stop the worker pools + webhookWorkerPool.Stop() + pollingWorkerPool.Stop() log.Println("Server exiting") } diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 0000000..d872c3e --- /dev/null +++ b/docs/DESIGN.md @@ -0,0 +1,17 @@ +# Design + +Instead of querying the GitHub API for each request, we will query our database. This will be faster and cheaper. At the organization level, a webhook is created that sends events to our server. + +## API + +### Endpoints + +#### GET /repositories/:id/workflows + +#### GET /workflows/:id/stats + +#### GET /workflows/:id/runs + +#### GET /runs/:id + +#### GET /jobs/:id diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index 1966218..a3bfa03 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -10,6 +10,247 @@ import ( "gorm.io/gorm" ) +// GetRepository returns a single repository by ID. +func GetRepository(c *gin.Context) { + repoIdParam := c.Param("id") + repoId, err := strconv.ParseInt(repoIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid repository ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var repo models.Repository + err = db.Where("id = ?", repoId).First(&repo).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve repository"}) + return + } + + c.JSON(http.StatusOK, repo) +} + +func GetRepositories(c *gin.Context) { + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var repos []models.Repository + err := db.Find(&repos).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve repositories"}) + return + } + + c.JSON(http.StatusOK, repos) +} + +// GetRepositoryWorkflows returns all workflows for a given repository. +func GetRepositoryWorkflows(c *gin.Context) { + repoIdParam := c.Param("id") + repoId, err := strconv.ParseInt(repoIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid repository ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var workflows []models.Workflow + err = db.Where("repository_id = ?", repoId).Find(&workflows).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve workflows"}) + return + } + + c.JSON(http.StatusOK, workflows) +} + +func GetWorkflow(c *gin.Context) { + workflowIdParam := c.Param("id") + workflowId, err := strconv.ParseInt(workflowIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid workflow ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var workflow models.Workflow + err = db.Where("id = ?", workflowId).First(&workflow).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve workflow"}) + return + } + + c.JSON(http.StatusOK, workflow) +} + +func GetWorkflowJobs(c *gin.Context) { + workflowIdParam := c.Param("id") + workflowId, err := strconv.ParseInt(workflowIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid workflow ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var jobs []models.Job + err = db.Where("workflow_id = ?", workflowId).Find(&jobs).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve workflow jobs"}) + return + } + + c.JSON(http.StatusOK, jobs) +} + +// GetWorkflowRuns returns all runs for a given workflow. +func GetWorkflowRuns(c *gin.Context) { + workflowIdParam := c.Param("id") + workflowId, err := strconv.ParseInt(workflowIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid workflow ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var runs []models.WorkflowRun + err = db.Where("workflow_id = ?", workflowId).Find(&runs).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve workflow runs"}) + return + } + + c.JSON(http.StatusOK, runs) +} + +// GetWorkflowRun returns a single workflow run by ID. +func GetWorkflowRun(c *gin.Context) { + runIdParam := c.Param("id") + runId, err := strconv.ParseInt(runIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid run ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var run models.WorkflowRun + err = db.Where("id = ?", runId).First(&run).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve workflow run"}) + return + } + + c.JSON(http.StatusOK, run) +} + +// GetJob returns a single job by ID. +func GetJob(c *gin.Context) { + jobIdParam := c.Param("id") + jobId, err := strconv.ParseInt(jobIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var job models.Job + err = db.Where("id = ?", jobId).First(&job).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve job"}) + return + } + + c.JSON(http.StatusOK, job) +} + +// GetJobSteps returns all steps for a given job. +func GetJobSteps(c *gin.Context) { + jobIdParam := c.Param("id") + jobId, err := strconv.ParseInt(jobIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var steps []models.TaskStep + err = db.Where("job_id = ?", jobId).Find(&steps).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve job steps"}) + return + } + + c.JSON(http.StatusOK, steps) +} + +// GetJobStats returns statistics for a given job. +func GetJobStats(c *gin.Context) { + jobIdParam := c.Param("id") + jobId, err := strconv.ParseInt(jobIdParam, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"}) + return + } + + db, ok := c.MustGet("db").(*gorm.DB) + if !ok { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database connection not found"}) + return + } + + var stats models.JobStatistics + err = db.Where("job_id = ?", jobId).First(&stats).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve job statistics"}) + return + } + + c.JSON(http.StatusOK, stats) +} + +// GetWorkflowStats returns statistics for a given workflow. func GetWorkflowStats(c *gin.Context) { workflowIDParam := c.Param("id") startTimeParam := c.Query("start_time") diff --git a/pkg/api/router.go b/pkg/api/router.go index ea1da35..517dcdd 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -12,18 +12,28 @@ import ( func StartServer(cfg *config.Config, db *db.Database, githubClient *github.Client, worker *worker.WorkerPool) { r := gin.Default() - // Public routes + // Public routes for Github OAuth r.GET("/login", auth.GitHubLogin) r.GET("/callback", auth.GitHubCallback) - // Webhook route (exclude middleware that could interfere) + // Webhook route for Github events (exclude middleware that could interfere) webhookHandler := github.NewWebhookHandler(db, githubClient, cfg.GitHub.WebhookSecret, worker) r.POST("/webhook", webhookHandler.HandleWebhook) - // Protected routes - protected := r.Group("/", auth.AuthMiddleware()) + // Require authentication for all repository routes + protected := r.Group("/repositories", auth.AuthMiddleware()) { - protected.GET("/workflows/:id/stats", GetWorkflowStats) + protected.GET("", GetRepositories) + protected.GET("/:repoId", GetRepository) + protected.GET("/:repoId/workflows", GetRepositoryWorkflows) // Get all workflows for a repository + protected.GET("/:repoId/workflows/:workflowId", GetWorkflow) // Get a specific workflow + protected.GET("/:repoId/workflows/:workflowId/runs", GetWorkflowRuns) // Get all runs for a workflow + protected.GET("/:repoId/workflows/:workflowId/runs/:runId", GetWorkflowRun) // Get a specific run + protected.GET("/:repoId/workflows/:workflowId/stats", GetWorkflowStats) // Get stats for a workflow + protected.GET("/:repoId/workflows/:workflowId/jobs", GetWorkflowJobs) // Get all jobs for a workflow + protected.GET("/:repoId/workflows/:workflowId/jobs/:jobId", GetJob) // Get a specific job + protected.GET("/:repoId/workflows/:workflowId/jobs/:jobId/steps", GetJobSteps) // Get all steps for a job + protected.GET("/:repoId/workflows/:workflowId/jobs/:jobId/stats", GetJobStats) // Get stats for a job } r.Run(":" + cfg.ServerPort) diff --git a/pkg/config/config.go b/pkg/config/config.go index 0c7f1c2..f000695 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,11 +21,12 @@ type DatabaseConfig struct { } type Config struct { - ServerPort string - LogLevel string - GitHub GitHubConfig - Database DatabaseConfig - WorkerPoolSize int + ServerPort string + LogLevel string + GitHub GitHubConfig + Database DatabaseConfig + PollingWorkerPoolSize int + WebhookWorkerPoolSize int } func LoadConfig() *Config { @@ -39,9 +40,10 @@ func LoadConfig() *Config { } return &Config{ - ServerPort: viper.GetString("server.port"), - LogLevel: viper.GetString("log.level"), - WorkerPoolSize: viper.GetInt("worker.pool_size"), + ServerPort: viper.GetString("server.port"), + LogLevel: viper.GetString("log.level"), + PollingWorkerPoolSize: viper.GetInt("polling_worker_pool_size"), + WebhookWorkerPoolSize: viper.GetInt("webhook_worker_pool_size"), GitHub: GitHubConfig{ ClientID: viper.GetString("github.client_id"), ClientSecret: viper.GetString("github.client_secret"), diff --git a/pkg/db/db.go b/pkg/db/db.go index 20dea7f..fe2cefc 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -25,7 +25,7 @@ func InitDB(cfg *config.Config) (*Database, error) { } // Auto-migrate the schema - err = conn.AutoMigrate(&models.Repository{}, &models.WorkflowRun{}, &models.Statistics{}) + err = conn.AutoMigrate(&models.Repository{}, &models.WorkflowRun{}, &models.WorkflowStatistics{}, &models.JobStatistics{}) if err != nil { return nil, fmt.Errorf("failed to auto-migrate schema: %w", err) } @@ -47,8 +47,21 @@ func (db *Database) GetRepositories() ([]models.Repository, error) { func (db *Database) SaveRepository(repo *models.Repository) error { repository := models.Repository{ - Name: repo.Name, - Owner: repo.Owner, + Name: repo.Name, + Owner: repo.Owner, + FullName: repo.FullName, + Description: repo.Description, + Private: repo.Private, + Fork: repo.Fork, + CreatedAt: repo.CreatedAt, + UpdatedAt: repo.UpdatedAt, + PushedAt: repo.PushedAt, + Size: repo.Size, + StarCount: repo.StarCount, + Language: repo.Language, + HasIssues: repo.HasIssues, + HasProjects: repo.HasProjects, + HasWiki: repo.HasWiki, } return db.Conn.Create(repository).Error } @@ -57,6 +70,35 @@ func (db *Database) DeleteRepository(id int) error { return db.Conn.Delete(&models.Repository{}, id).Error } +func (db *Database) GetWorkflow(id int) (*models.Workflow, error) { + var workflow models.Workflow + err := db.Conn.First(&workflow, id).Error + return &workflow, err +} + +func (db *Database) GetWorkflows() ([]models.Workflow, error) { + var workflows []models.Workflow + err := db.Conn.Find(&workflows).Error + return workflows, err +} + +func (db *Database) SaveWorkflow(workflow *github.Workflow) error { + workflowModel := models.Workflow{ + WorkflowID: workflow.GetID(), + NodeID: workflow.GetNodeID(), + Name: workflow.GetName(), + Path: workflow.GetPath(), + State: workflow.GetState(), + CreatedAt: workflow.GetCreatedAt().Time, + UpdatedAt: workflow.GetUpdatedAt().Time, + } + return db.Conn.Create(workflowModel).Error +} + +func (db *Database) DeleteWorkflow(id int) error { + return db.Conn.Delete(&models.Workflow{}, id).Error +} + func (db *Database) GetWorkflowRun(id int) (*models.WorkflowRun, error) { var run models.WorkflowRun err := db.Conn.First(&run, id).Error @@ -92,42 +134,79 @@ func (db *Database) DeleteWorkflowRun(id int) error { return db.Conn.Delete(&models.WorkflowRun{}, id).Error } -func (db *Database) GetWorkflow(id int) (*models.Workflow, error) { - var workflow models.Workflow - err := db.Conn.First(&workflow, id).Error - return &workflow, err +func (db *Database) GetWorkflowJob(workflowID int) (*models.Job, error) { + var job models.Job + err := db.Conn.First(&job, workflowID).Error + return &job, err +} + +func (db *Database) GetWorkflowJobs(workflowID int) ([]models.Job, error) { + var jobs []models.Job + err := db.Conn.Where("workflow_id = ?", workflowID).Find(&jobs).Error + return jobs, err +} + +func (db *Database) SaveWorkflowJob(job *github.WorkflowJob) error { + jobModel := models.Job{ + JobID: job.GetID(), + RunID: job.GetRunID(), + RunURL: job.GetRunURL(), + NodeID: job.GetNodeID(), + HeadSHA: job.GetHeadSHA(), + URL: job.GetURL(), + HTMLURL: job.GetHTMLURL(), + Status: job.GetStatus(), + Conclusion: job.GetConclusion(), + CreatedAt: job.GetCreatedAt().Time, + CompletedAt: job.GetCompletedAt().Time, + Name: job.GetName(), + Steps: []models.TaskStep{}, + CheckRunURL: job.GetCheckRunURL(), + Labels: job.Labels, + RunnerID: job.GetRunnerID(), + RunnerName: job.GetRunnerName(), + RunnerGroupID: job.GetRunnerGroupID(), + RunnerGroupName: job.GetRunnerGroupName(), + RunAttempt: int(job.GetRunAttempt()), + WorkflowName: job.GetWorkflowName(), + } + return db.Conn.Create(jobModel).Error } -func (db *Database) GetWorkflows() ([]models.Workflow, error) { - var workflows []models.Workflow - err := db.Conn.Find(&workflows).Error - return workflows, err +func (db *Database) DeleteWorkflowJob(id int) error { + return db.Conn.Delete(&models.Job{}, id).Error } -func (db *Database) SaveWorkflow(workflow *models.Workflow) error { - workflowModel := models.Workflow{ - Name: workflow.Name, +func (db *Database) GetJobStatistics() ([]models.JobStatistics, error) { + var stats []models.JobStatistics + err := db.Conn.Find(&stats).Error + return stats, err +} + +func (db *Database) SaveJobStatistics(stats *models.JobStatistics) error { + jobStatistics := models.JobStatistics{ + ID: stats.ID, } - return db.Conn.Create(workflowModel).Error + return db.Conn.Save(jobStatistics).Error } -func (db *Database) DeleteWorkflow(id int) error { - return db.Conn.Delete(&models.Workflow{}, id).Error +func (db *Database) DeleteJobStatistics(id int) error { + return db.Conn.Delete(&models.JobStatistics{}, id).Error } -func (db *Database) GetStatistics() ([]models.Statistics, error) { - var stats []models.Statistics +func (db *Database) GetWorkflowStatistics() ([]models.WorkflowStatistics, error) { + var stats []models.WorkflowStatistics err := db.Conn.Find(&stats).Error return stats, err } -func (db *Database) SaveStatistics(stats *models.Statistics) error { - workflowStatistics := models.Statistics{ +func (db *Database) SaveWorkflowStatistics(stats *models.WorkflowStatistics) error { + workflowStatistics := models.WorkflowStatistics{ ID: stats.ID, } return db.Conn.Save(workflowStatistics).Error } -func (db *Database) DeleteStatistics(id int) error { - return db.Conn.Delete(&models.Statistics{}, id).Error +func (db *Database) DeleteWorkflowStatistics(id int) error { + return db.Conn.Delete(&models.WorkflowStatistics{}, id).Error } diff --git a/pkg/db/models/job.go b/pkg/db/models/job.go new file mode 100644 index 0000000..abc62c3 --- /dev/null +++ b/pkg/db/models/job.go @@ -0,0 +1,35 @@ +package models + +import ( + "time" + + "gorm.io/gorm" +) + +type Job struct { + gorm.Model + ID int64 + JobID int64 + RunID int64 + RunURL string + NodeID string + HeadSHA string + URL string + HTMLURL string + LogsURL string + CheckRunURL string + RunnerID int64 + CreatedAt time.Time + Name string + Labels []string + RunAttempt int + RunnerName string + RunnerGroupID int64 + RunnerGroupName string + WorkflowID int64 + WorkflowName string + Status string + Conclusion string + CompletedAt time.Time + Steps []TaskStep +} diff --git a/pkg/db/models/statistics.go b/pkg/db/models/statistics.go index 88b4631..3158cd5 100644 --- a/pkg/db/models/statistics.go +++ b/pkg/db/models/statistics.go @@ -2,10 +2,15 @@ package models import "time" -type Statistics struct { +type WorkflowStatistics struct { ID uint `gorm:"primaryKey"` TotalRuns int64 SuccessRate float64 // Add fields for additional statistics UpdatedAt time.Time } + +type JobStatistics struct { + ID uint `gorm:"primaryKey"` + UpdatedAt time.Time +} diff --git a/pkg/db/models/step.go b/pkg/db/models/step.go new file mode 100644 index 0000000..2089230 --- /dev/null +++ b/pkg/db/models/step.go @@ -0,0 +1,16 @@ +package models + +import ( + "time" + + "gorm.io/gorm" +) + +type TaskStep struct { + gorm.Model + Name string + Status string + Conclusion string + StartedAt time.Time + CompletedAt time.Time +} diff --git a/pkg/db/models/workflow.go b/pkg/db/models/workflow.go index 7ea18de..9c84807 100644 --- a/pkg/db/models/workflow.go +++ b/pkg/db/models/workflow.go @@ -9,6 +9,8 @@ import ( // Workflow represents a GitHub workflow type Workflow struct { gorm.Model + WorkflowID int64 `gorm:"index"` + NodeID string `gorm:"index"` Name string `gorm:"type:varchar(255);not null"` Path string `gorm:"type:varchar(255);not null"` State string `gorm:"type:varchar(50)"` @@ -19,7 +21,7 @@ type Workflow struct { BadgeURL string `gorm:"column:badge_url;type:varchar(255)"` RepositoryID uint `gorm:"not null"` // You might want to add a foreign key relationship to a Repository model if you have one - // Repository Repository `gorm:"foreignKey:RepositoryID"` + Repository Repository `gorm:"foreignKey:RepositoryID"` } // TableName specifies the table name for the Workflow model diff --git a/pkg/db/models/workflow_run.go b/pkg/db/models/workflow_run.go index 90f3233..d933666 100644 --- a/pkg/db/models/workflow_run.go +++ b/pkg/db/models/workflow_run.go @@ -11,6 +11,7 @@ type WorkflowRun struct { gorm.Model WorkflowID int64 `gorm:"index"` Name string + NodeID string HeadBranch string HeadSHA string Status string diff --git a/pkg/github/polling.go b/pkg/github/polling.go index 1f2a5a4..4218cd2 100644 --- a/pkg/github/polling.go +++ b/pkg/github/polling.go @@ -47,23 +47,49 @@ func (p *Poller) Start() { } } -// pollRepositories fetches all repositories from the database and polls their workflows concurrently. +// pollRepositories fetches all repositories accessible to the authenticated user and polls their workflows concurrently. func (p *Poller) pollRepositories() { - repos, err := p.db.GetRepositories() - if err != nil { - log.Printf("Error fetching repositories: %v", err) - return + opt := &gh.RepositoryListOptions{ + ListOptions: gh.ListOptions{PerPage: 10}, // Adjust per page as needed + } + + var allRepos []*gh.Repository + for { + repos, resp, err := p.ghClient.Repositories.List(context.Background(), "", opt) + if err != nil { + log.Printf("Error fetching repositories: %v", err) + return + } + allRepos = append(allRepos, repos...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage } var wg sync.WaitGroup sem := make(chan struct{}, maxConcurrentPolls) - for _, repo := range repos { + for _, repo := range allRepos { wg.Add(1) sem <- struct{}{} - go func(repo models.Repository) { + go func(repo *gh.Repository) { defer wg.Done() - p.pollWorkflows(repo) + dbRepo := models.Repository{ + Owner: models.GitHubUser{ + Email: func(email *string) string { + if email != nil { + return *email + } + return "" + }(repo.Owner.Email), + }, + Name: *repo.Name, // Dereference the pointer + } + err := p.db.SaveRepository(&dbRepo) + if err != nil { + log.Printf("Error saving repository %s: %v", repo.Name, err) + } <-sem }(repo) } diff --git a/pkg/github/webhook.go b/pkg/github/webhook.go index 269b355..a437cba 100644 --- a/pkg/github/webhook.go +++ b/pkg/github/webhook.go @@ -70,8 +70,11 @@ func (wh *WebhookHandler) HandleWebhook(c *gin.Context) { // Handle different event types switch e := event.(type) { - case *github.WorkflowRunEvent: + case *github.WorkflowRunEvent: // WorkflowRunEvent is triggered when a GitHub Actions workflow run is requested or completed. wh.handleWorkflowRunEvent(e) + case *github.WorkflowJobEvent: // WorkflowJobEvent is triggered when a job is queued, started or completed. + wh.handleWorkflowJobEvent(e) + default: // Unsupported event type c.Status(http.StatusOK) @@ -106,12 +109,17 @@ func (wh *WebhookHandler) verifySignature(signature string, payload []byte) bool // - event: A pointer to the GitHub WorkflowRunEvent. func (wh *WebhookHandler) handleWorkflowRunEvent(event *github.WorkflowRunEvent) { action := event.GetAction() + workflow := event.GetWorkflow() run := event.GetWorkflowRun() switch action { case "completed": + err := wh.db.SaveWorkflow(workflow) + if err != nil { + // Log error + } // Save or update the workflow run in the database - err := wh.db.SaveWorkflowRun(run) + err = wh.db.SaveWorkflowRun(run) if err != nil { // Log error } @@ -125,3 +133,11 @@ func (wh *WebhookHandler) handleWorkflowRunEvent(event *github.WorkflowRunEvent) // Handle other actions if needed } } + +func (wh *WebhookHandler) handleWorkflowJobEvent(event *github.WorkflowJobEvent) { + job := event.GetWorkflowJob() + err := wh.db.SaveWorkflowJob(job) + if err != nil { + // Log error + } +}