diff --git a/pkg/domain/constant/global.go b/pkg/domain/constant/global.go index 6bc391e8b..1afd90338 100644 --- a/pkg/domain/constant/global.go +++ b/pkg/domain/constant/global.go @@ -19,4 +19,9 @@ const ( RepoCacheTTL = 60 * time.Minute RunTimeOut = 60 * time.Minute DefaultWorkloadSig = "kusion.io/is-workload" + ResourcePageDefault = 1 + ResourcePageSizeDefault = 100 + ResourcePageSizeLarge = 1000 + RunPageDefault = 1 + RunPageSizeDefault = 10 ) diff --git a/pkg/domain/constant/organization.go b/pkg/domain/constant/organization.go index 6101d52ae..17bd9ba49 100644 --- a/pkg/domain/constant/organization.go +++ b/pkg/domain/constant/organization.go @@ -22,6 +22,7 @@ var ( ErrBackendNil = errors.New("backend is nil") ErrBackendNameEmpty = errors.New("backend must have a name") ErrBackendTypeEmpty = errors.New("backend must have a type") + ErrDefaultBackendNotSet = errors.New("default backend not set properly") ErrAppConfigHasNilStack = errors.New("appConfig has nil stack") ErrInvalidOrganizationName = errors.New("organization name can only have alphanumeric characters and underscores with [a-zA-Z0-9_]") ErrInvalidOrganizationID = errors.New("the organization ID should be a uuid") diff --git a/pkg/domain/entity/run.go b/pkg/domain/entity/run.go index 5728534ae..34142e98a 100644 --- a/pkg/domain/entity/run.go +++ b/pkg/domain/entity/run.go @@ -22,7 +22,8 @@ type Run struct { Status constant.RunStatus `yaml:"status" json:"status"` // Result is the result of the run. Result string `yaml:"result" json:"result"` - // Result RunResult `yaml:"result" json:"result"` + // Trace is the trace of the run. + Trace string `yaml:"trace" json:"trace"` // Logs is the logs of the run. Logs string `yaml:"logs" json:"logs"` // CreationTimestamp is the timestamp of the created for the run. @@ -44,9 +45,15 @@ type RunResult struct { } type RunFilter struct { - ProjectID uint - StackID uint - Workspace string + ProjectID uint + StackID uint + Workspace string + Pagination *Pagination +} + +type RunListResult struct { + Runs []*Run + Total int } // Validate checks if the run is valid. diff --git a/pkg/domain/entity/types.go b/pkg/domain/entity/types.go new file mode 100644 index 000000000..96ae0837d --- /dev/null +++ b/pkg/domain/entity/types.go @@ -0,0 +1,6 @@ +package entity + +type Pagination struct { + Page int `json:"page"` + PageSize int `json:"pageSize"` +} diff --git a/pkg/domain/repository/repository.go b/pkg/domain/repository/repository.go index e0088d8a0..d94f45602 100644 --- a/pkg/domain/repository/repository.go +++ b/pkg/domain/repository/repository.go @@ -150,5 +150,5 @@ type RunRepository interface { // Get retrieves a run by its ID. Get(ctx context.Context, id uint) (*entity.Run, error) // List retrieves all existing run. - List(ctx context.Context, filter *entity.RunFilter) ([]*entity.Run, error) + List(ctx context.Context, filter *entity.RunFilter) (*entity.RunListResult, error) } diff --git a/pkg/domain/response/resource.go b/pkg/domain/response/resource.go new file mode 100644 index 000000000..a347af7a8 --- /dev/null +++ b/pkg/domain/response/resource.go @@ -0,0 +1,12 @@ +package response + +import ( + "kusionstack.io/kusion/pkg/domain/entity" +) + +type PaginatedResourceResponse struct { + Resources []*entity.Resource `json:"resources"` + Total int `json:"total"` + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` +} diff --git a/pkg/domain/response/run.go b/pkg/domain/response/run.go new file mode 100644 index 000000000..bb6884407 --- /dev/null +++ b/pkg/domain/response/run.go @@ -0,0 +1,12 @@ +package response + +import ( + "kusionstack.io/kusion/pkg/domain/entity" +) + +type PaginatedRunResponse struct { + Runs []*entity.Run `json:"runs"` + Total int `json:"total"` + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` +} diff --git a/pkg/infra/persistence/run.go b/pkg/infra/persistence/run.go index 5b9757b6f..5a660fa87 100644 --- a/pkg/infra/persistence/run.go +++ b/pkg/infra/persistence/run.go @@ -96,17 +96,24 @@ func (r *runRepository) Get(ctx context.Context, id uint) (*entity.Run, error) { } // List retrieves all runs. -func (r *runRepository) List(ctx context.Context, filter *entity.RunFilter) ([]*entity.Run, error) { +func (r *runRepository) List(ctx context.Context, filter *entity.RunFilter) (*entity.RunListResult, error) { var dataModel []RunModel runEntityList := make([]*entity.Run, 0) pattern, args := GetRunQuery(filter) - result := r.db.WithContext(ctx). + searchResult := r.db.WithContext(ctx). Preload("Stack").Preload("Stack.Project"). Joins("JOIN stack ON stack.id = run.stack_id"). Joins("JOIN project ON project.id = stack.project_id"). Joins("JOIN workspace ON workspace.name = run.workspace"). - Where(pattern, args...). - Find(&dataModel) + Where(pattern, args...) + + // Get total rows + var totalRows int64 + searchResult.Model(dataModel).Count(&totalRows) + + // Fetch paginated data from searchResult with offset and limit + offset := (filter.Pagination.Page - 1) * filter.Pagination.PageSize + result := searchResult.Offset(offset).Limit(filter.Pagination.PageSize).Find(&dataModel) if result.Error != nil { return nil, result.Error } @@ -117,5 +124,8 @@ func (r *runRepository) List(ctx context.Context, filter *entity.RunFilter) ([]* } runEntityList = append(runEntityList, runEntity) } - return runEntityList, nil + return &entity.RunListResult{ + Runs: runEntityList, + Total: int(totalRows), + }, nil } diff --git a/pkg/infra/persistence/run_model.go b/pkg/infra/persistence/run_model.go index 48f7f9464..bb855301c 100644 --- a/pkg/infra/persistence/run_model.go +++ b/pkg/infra/persistence/run_model.go @@ -24,6 +24,8 @@ type RunModel struct { Result string // Logs is the logs of the run. Logs string + // Trace is the trace of the run. + Trace string } // The TableName method returns the name of the database table that the struct is mapped to. @@ -53,13 +55,13 @@ func (m *RunModel) ToEntity() (*entity.Run, error) { } return &entity.Run{ - ID: m.ID, - Type: runType, - Stack: stackEntity, - Workspace: m.Workspace, - Status: runStatus, - Result: m.Result, - // Result: entity.RunResult{}, + ID: m.ID, + Type: runType, + Stack: stackEntity, + Workspace: m.Workspace, + Status: runStatus, + Result: m.Result, + Trace: m.Trace, Logs: m.Logs, CreationTimestamp: m.CreatedAt, UpdateTimestamp: m.UpdatedAt, @@ -83,6 +85,7 @@ func (m *RunModel) FromEntity(e *entity.Run) error { m.Status = string(e.Status) m.Result = e.Result m.Logs = e.Logs + m.Trace = e.Trace m.CreatedAt = e.CreationTimestamp m.UpdatedAt = e.UpdateTimestamp diff --git a/pkg/server/handler/stack/execute_async.go b/pkg/server/handler/stack/execute_async.go index d07fce825..0645fb621 100644 --- a/pkg/server/handler/stack/execute_async.go +++ b/pkg/server/handler/stack/execute_async.go @@ -103,14 +103,12 @@ func (h *Handler) PreviewStackAsync() http.HandlerFunc { // Call preview stack changes, err := h.stackManager.PreviewStack(newCtx, params, requestPayload.ImportedResources) if err != nil { - // render.Render(w, r, handler.FailureResponse(ctx, err)) logger.Error("Error previewing stack", "error", err) return } previewChanges, err = stackmanager.ProcessChanges(newCtx, w, changes, params.Format, params.ExecuteParams.Detail) if err != nil { - // render.Render(w, r, handler.FailureResponse(ctx, err)) logger.Error("Error processing preview changes", "error", err) return } @@ -206,7 +204,6 @@ func (h *Handler) ApplyStackAsync() http.HandlerFunc { render.Render(w, r, handler.SuccessResponse(ctx, "Dry-run mode enabled, the above resources will be applied if dryrun is set to false")) return } else { - // render.Render(w, r, handler.FailureResponse(ctx, err)) logger.Error("Error applying stack", "error", err) return } @@ -308,9 +305,8 @@ func (h *Handler) GenerateStackAsync() http.HandlerFunc { }() // Call generate stack - _, sp, err := h.stackManager.GenerateSpec(newCtx, params) + _, sp, err = h.stackManager.GenerateSpec(newCtx, params) if err != nil { - // render.Render(w, r, handler.FailureResponse(ctx, err)) logger.Error("Error generating stack", "error", err) return } @@ -400,11 +396,9 @@ func (h *Handler) DestroyStackAsync() http.HandlerFunc { err = h.stackManager.DestroyStack(newCtx, params, w) if err != nil { if err == stackmanager.ErrDryrunDestroy { - // render.Render(w, r, handler.SuccessResponse(ctx, "Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false")) logger.Info("Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false") return } else { - // render.Render(w, r, handler.FailureResponse(ctx, err)) logger.Error("Error destroying stack", "error", err) return } diff --git a/pkg/server/handler/stack/run.go b/pkg/server/handler/stack/run.go index ed7f61e6c..92ae131d1 100644 --- a/pkg/server/handler/stack/run.go +++ b/pkg/server/handler/stack/run.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/go-chi/render" + response "kusionstack.io/kusion/pkg/domain/response" "kusionstack.io/kusion/pkg/server/handler" logutil "kusionstack.io/kusion/pkg/server/util/logging" ) @@ -95,18 +96,26 @@ func (h *Handler) ListRuns() http.HandlerFunc { logger := logutil.GetLogger(ctx) logger.Info("Listing runs...") - projectIDParam := r.URL.Query().Get("projectID") - stackIDParam := r.URL.Query().Get("stackID") - workspaceParam := r.URL.Query().Get("workspace") - - filter, err := h.stackManager.BuildRunFilter(ctx, projectIDParam, stackIDParam, workspaceParam) + query := r.URL.Query() + filter, err := h.stackManager.BuildRunFilter(ctx, &query) if err != nil { render.Render(w, r, handler.FailureResponse(ctx, err)) return } + // List runs runEntities, err := h.stackManager.ListRuns(ctx, filter) - handler.HandleResult(w, r, ctx, err, runEntities) + if err != nil { + render.Render(w, r, handler.FailureResponse(ctx, err)) + return + } + paginatedResponse := response.PaginatedRunResponse{ + Runs: runEntities.Runs, + Total: runEntities.Total, + CurrentPage: filter.Pagination.Page, + PageSize: filter.Pagination.PageSize, + } + handler.HandleResult(w, r, ctx, err, paginatedResponse) } } diff --git a/pkg/server/manager/stack/execute.go b/pkg/server/manager/stack/execute.go index 113df23a1..04d85752f 100644 --- a/pkg/server/manager/stack/execute.go +++ b/pkg/server/manager/stack/execute.go @@ -26,7 +26,8 @@ import ( func (m *StackManager) GenerateSpec(ctx context.Context, params *StackRequestParams) (string, *apiv1.Spec, error) { logger := logutil.GetLogger(ctx) - logger.Info("Starting generating spec in StackManager ...") + runLogger := logutil.GetRunLogger(ctx) + logToAll(logger, runLogger, "Info", "Starting generating spec in StackManager...") // Get the stack entity and return error if stack ID is not found stackEntity, err := m.stackRepo.Get(ctx, params.StackID) @@ -64,6 +65,9 @@ func (m *StackManager) GenerateSpec(ctx context.Context, params *StackRequestPar // Otherwise, generate spec from stack entity using the default generator project, stack, wsBackend, err := m.getStackProjectAndBackend(ctx, stackEntity, params.Workspace) + if err != nil { + return "", nil, err + } wsStorage, err := wsBackend.WorkspaceStorage() if err != nil { return "", nil, err @@ -99,7 +103,8 @@ func (m *StackManager) GenerateSpec(ctx context.Context, params *StackRequestPar func (m *StackManager) PreviewStack(ctx context.Context, params *StackRequestParams, requestPayload request.StackImportRequest) (*models.Changes, error) { logger := logutil.GetLogger(ctx) - logger.Info("Starting previewing stack in StackManager ...") + runLogger := logutil.GetRunLogger(ctx) + logToAll(logger, runLogger, "Info", "Starting previewing stack in StackManager...") // Get the stack entity by id stackEntity, err := m.stackRepo.Get(ctx, params.StackID) @@ -112,7 +117,7 @@ func (m *StackManager) PreviewStack(ctx context.Context, params *StackRequestPar defer func() { if err != nil { - logger.Info("Error occurred during previewing stack. Setting stack sync state to preview failed") + logToAll(logger, runLogger, "Info", "Error occurred during previewing stack. Setting stack sync state to preview failed") stackEntity.SyncState = constant.StackStatePreviewFailed m.stackRepo.Update(ctx, stackEntity) } else { @@ -161,7 +166,7 @@ func (m *StackManager) PreviewStack(ctx context.Context, params *StackRequestPar if err != nil { return nil, err } - logger.Info("State storage found with path", "releasePath", releasePath) + logToAll(logger, runLogger, "Info", "State storage found with path", "releasePath", releasePath) directory, workDir, err := m.GetWorkdirAndDirectory(ctx, params, stackEntity) if err != nil { @@ -185,7 +190,7 @@ func (m *StackManager) PreviewStack(ctx context.Context, params *StackRequestPar // return immediately if no resource found in stack // todo: if there is no resource, should still do diff job; for now, if output is json format, there is no hint if sp == nil || len(sp.Resources) == 0 { - logger.Info("No resource change found in this stack...") + logToAll(logger, runLogger, "Info", "No resource change found in this stack...") return nil, nil } @@ -208,7 +213,7 @@ func (m *StackManager) PreviewStack(ctx context.Context, params *StackRequestPar if params.ExecuteParams.ImportResources && len(requestPayload.ImportedResources) > 0 { m.ImportTerraformResourceID(ctx, sp, requestPayload.ImportedResources) } - logger.Info("Final Spec is: ", "spec", sp) + logToAll(logger, runLogger, "Info", "Final Spec is: ", "spec", sp) changes, err := engineapi.Preview(executeOptions, releaseStorage, sp, state, project, stack) return changes, err @@ -216,7 +221,8 @@ func (m *StackManager) PreviewStack(ctx context.Context, params *StackRequestPar func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParams, requestPayload request.StackImportRequest) error { logger := logutil.GetLogger(ctx) - logger.Info("Starting applying stack in StackManager ...") + runLogger := logutil.GetRunLogger(ctx) + logToAll(logger, runLogger, "Info", "Starting applying stack in StackManager ...") _, stackBackend, project, _, ws, err := m.metaHelper(ctx, params.StackID, params.Workspace) if err != nil { return err @@ -235,10 +241,10 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam // If specID is explicitly specified by the caller, use the spec with the specID if params.ExecuteParams.SpecID != "" { specID = params.ExecuteParams.SpecID - logger.Info("SpecID explicitly set. Using the specified version", "SpecID", specID) + logToAll(logger, runLogger, "Info", "SpecID explicitly set. Using the specified version", "SpecID", specID) } else { specID = stackEntity.LastPreviewedRevision - logger.Info("SpecID not explicitly set. Using last previewed version", "SpecID", stackEntity.LastPreviewedRevision) + logToAll(logger, runLogger, "Info", "SpecID not explicitly set. Using last previewed version", "SpecID", stackEntity.LastPreviewedRevision) } var storage release.Storage @@ -294,7 +300,7 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam if err != nil { return err } - logger.Info("State storage found with path", "releasePath", releasePath) + logToAll(logger, runLogger, "Info", "State storage found with path", "releasePath", releasePath) if err != nil { return err } @@ -320,9 +326,12 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam var sp *apiv1.Spec var changes *models.Changes project, stack, stateBackend, err := m.getStackProjectAndBackend(ctx, stackEntity, params.Workspace) + if err != nil { + return err + } executeOptions := BuildOptions(params.ExecuteParams.Dryrun, m.maxConcurrent) - logger.Info("Previewing using the default generator ...") + logToAll(logger, runLogger, "Info", "Previewing using the default generator ...") directory, workDir, err := m.GetWorkdirAndDirectory(ctx, params, stackEntity) if err != nil { @@ -346,7 +355,7 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam // return immediately if no resource found in stack // todo: if there is no resource, should still do diff job; for now, if output is json format, there is no hint if sp == nil || len(sp.Resources) == 0 { - logger.Info("No resource change found in this stack...") + logToAll(logger, runLogger, "Info", "No resource change found in this stack...") return nil } @@ -359,12 +368,12 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam // if dry run, print the hint if params.ExecuteParams.Dryrun { - logger.Info("NOTE: Currently running in the --dry-run mode, the above configuration does not really take effect") + logToAll(logger, runLogger, "Info", "Dry-run mode enabled, the above resources will be applied if dryrun is set to false") err = ErrDryrunApply return err } - logger.Info("State backend found", "stateBackend", stateBackend) + logToAll(logger, runLogger, "Info", "State backend found", "stateBackend", stateBackend) stack.Path = tempPath(stackEntity.Path) // Set context from workspace to spec @@ -386,7 +395,7 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam return err } - logger.Info("Dryrun set to false. Start applying diffs ...") + logToAll(logger, runLogger, "Info", "Start applying diffs ...") release.UpdateReleasePhase(rel, apiv1.ReleasePhaseApplying, relLock) if err = release.UpdateApplyRelease(storage, rel, params.ExecuteParams.Dryrun, relLock); err != nil { return err @@ -445,7 +454,8 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam func (m *StackManager) DestroyStack(ctx context.Context, params *StackRequestParams, w http.ResponseWriter) (err error) { logger := logutil.GetLogger(ctx) - logger.Info("Starting destroying stack in StackManager ...") + runLogger := logutil.GetRunLogger(ctx) + logToAll(logger, runLogger, "Info", "Starting destroying stack in StackManager ...") // Get the stack entity by id stackEntity, err := m.stackRepo.Get(ctx, params.StackID) @@ -503,7 +513,7 @@ func (m *StackManager) DestroyStack(ctx context.Context, params *StackRequestPar if err != nil { return err } - logger.Info("State storage found with path", "releasePath", releasePath) + logToAll(logger, runLogger, "Info", "State storage found with path", "releasePath", releasePath) if err != nil { return err } @@ -534,7 +544,7 @@ func (m *StackManager) DestroyStack(ctx context.Context, params *StackRequestPar // if dryrun, print the hint if params.ExecuteParams.Dryrun { - logger.Info("Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false") + logToAll(logger, runLogger, "Info", "Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false") return ErrDryrunDestroy } @@ -544,7 +554,7 @@ func (m *StackManager) DestroyStack(ctx context.Context, params *StackRequestPar return } // Destroy - logger.Info("Start destroying resources......") + logToAll(logger, runLogger, "Info", "Start destroying resources......") var upRel *apiv1.Release upRel, err = engineapi.Destroy(executeOptions, rel, changes, storage) diff --git a/pkg/server/manager/stack/run.go b/pkg/server/manager/stack/run.go index 009c03313..a4b290be7 100644 --- a/pkg/server/manager/stack/run.go +++ b/pkg/server/manager/stack/run.go @@ -12,10 +12,11 @@ import ( "kusionstack.io/kusion/pkg/domain/entity" "kusionstack.io/kusion/pkg/domain/request" + appmiddleware "kusionstack.io/kusion/pkg/server/middleware" logutil "kusionstack.io/kusion/pkg/server/util/logging" ) -func (m *StackManager) ListRuns(ctx context.Context, filter *entity.RunFilter) ([]*entity.Run, error) { +func (m *StackManager) ListRuns(ctx context.Context, filter *entity.RunFilter) (*entity.RunListResult, error) { runEntities, err := m.runRepo.List(ctx, filter) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { @@ -99,6 +100,9 @@ func (m *StackManager) CreateRun(ctx context.Context, requestPayload request.Cre // The default status is InProgress createdEntity.Status = constant.RunStatusInProgress + // Inject trace into run metadata + traceID := appmiddleware.GetTraceID(ctx) + createdEntity.Trace = traceID // Create run with repository err = m.runRepo.Create(ctx, &createdEntity) if err != nil && err == gorm.ErrDuplicatedKey { diff --git a/pkg/server/manager/stack/util.go b/pkg/server/manager/stack/util.go index 0eb0ca986..daf01257e 100644 --- a/pkg/server/manager/stack/util.go +++ b/pkg/server/manager/stack/util.go @@ -5,12 +5,14 @@ import ( "errors" "fmt" "net/http" + "net/url" "os" "path/filepath" "regexp" "strconv" "strings" + "github.com/go-chi/httplog/v2" "gorm.io/gorm" v1 "kusionstack.io/kusion/pkg/apis/api.kusion.io/v1" "kusionstack.io/kusion/pkg/backend" @@ -135,6 +137,9 @@ func (m *StackManager) getBackendFromWorkspaceName(ctx context.Context, workspac var remoteBackend backend.Backend if workspaceName == constant.DefaultWorkspace { // Get default backend + if m.defaultBackend.BackendConfig.Type == "" { + return nil, constant.ErrDefaultBackendNotSet + } return m.getDefaultBackend() } else { // Get backend by id @@ -267,9 +272,14 @@ func (m *StackManager) BuildStackFilter(ctx context.Context, orgIDParam, project return &filter, nil } -func (m *StackManager) BuildRunFilter(ctx context.Context, projectIDParam, stackIDParam, workspaceParam string) (*entity.RunFilter, error) { +func (m *StackManager) BuildRunFilter(ctx context.Context, query *url.Values) (*entity.RunFilter, error) { logger := logutil.GetLogger(ctx) logger.Info("Building run filter...") + + projectIDParam := query.Get("projectID") + stackIDParam := query.Get("stackID") + workspaceParam := query.Get("workspace") + filter := entity.RunFilter{} if projectIDParam != "" { // if project id is present, use project id @@ -291,6 +301,19 @@ func (m *StackManager) BuildRunFilter(ctx context.Context, projectIDParam, stack // if workspace is present, use workspace filter.Workspace = workspaceParam } + // Set pagination parameters + page, _ := strconv.Atoi(query.Get("page")) + if page <= 0 { + page = constant.RunPageDefault + } + pageSize, _ := strconv.Atoi(query.Get("pageSize")) + if pageSize <= 0 { + pageSize = constant.RunPageSizeDefault + } + filter.Pagination = &entity.Pagination{ + Page: page, + PageSize: pageSize, + } return &filter, nil } @@ -411,3 +434,22 @@ func isInRelease(release *v1.Release, id string) bool { } return false } + +func logToAll(sysLogger *httplog.Logger, runLogger *httplog.Logger, level string, message string, args ...any) { + switch strings.ToLower(level) { + case "info": + sysLogger.Info(message, args...) + runLogger.Info(message, args...) + case "error": + sysLogger.Error(message, args...) + runLogger.Error(message, args...) + case "warn": + sysLogger.Warn(message, args...) + runLogger.Warn(message, args...) + case "debug": + sysLogger.Debug(message, args...) + runLogger.Debug(message, args...) + default: + sysLogger.Error("unknown log level", "level", level) + } +}