From 4d2c49c0df8bd0ea975240afbc402d15b7499c5b Mon Sep 17 00:00:00 2001 From: Dayuan Date: Wed, 9 Aug 2023 19:11:41 +0800 Subject: [PATCH] feat: add AppConfiguration, Deploy, and Job models (#451) --- .github/workflows/constraint.yaml | 41 +++++ .github/workflows/test.yaml | 11 +- .golangci.yml | 4 +- commitlint.config.js | 21 +++ pkg/cmd/preview/options_test.go | 3 +- pkg/cmd/spec/generator.go | 38 +++- .../app_configuration_generator.go | 33 ++++ .../generators/app_configuration_generator.go | 45 +++++ .../generators/components_generator.go | 55 ++++++ .../generators/deployment_generator.go | 112 ++++++++++++ .../generators/job_generator.go | 98 ++++++++++ .../generators/job_generator_test.go | 163 +++++++++++++++++ .../generators/namespace_generator.go | 50 +++++ .../appconfiguration/generators/types.go | 9 + .../appconfiguration/generators/util.go | 171 ++++++++++++++++++ pkg/generator/generator.go | 2 +- pkg/generator/kcl/kcl_generator.go | 2 +- pkg/generator/kcl/kcl_generator_test.go | 19 +- .../appconfiguration/appconfiguration.go | 33 ++++ .../appconfiguration/component/component.go | 22 +++ .../component/container/container.go | 17 ++ .../component/workload/job.go | 12 ++ .../workload/long_running_service.go | 13 ++ pkg/projectstack/types.go | 15 +- 24 files changed, 959 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/constraint.yaml create mode 100644 commitlint.config.js create mode 100644 pkg/generator/appconfiguration/app_configuration_generator.go create mode 100644 pkg/generator/appconfiguration/generators/app_configuration_generator.go create mode 100644 pkg/generator/appconfiguration/generators/components_generator.go create mode 100644 pkg/generator/appconfiguration/generators/deployment_generator.go create mode 100644 pkg/generator/appconfiguration/generators/job_generator.go create mode 100644 pkg/generator/appconfiguration/generators/job_generator_test.go create mode 100644 pkg/generator/appconfiguration/generators/namespace_generator.go create mode 100644 pkg/generator/appconfiguration/generators/types.go create mode 100644 pkg/generator/appconfiguration/generators/util.go create mode 100644 pkg/models/appconfiguration/appconfiguration.go create mode 100644 pkg/models/appconfiguration/component/component.go create mode 100644 pkg/models/appconfiguration/component/container/container.go create mode 100644 pkg/models/appconfiguration/component/workload/job.go create mode 100644 pkg/models/appconfiguration/component/workload/long_running_service.go diff --git a/.github/workflows/constraint.yaml b/.github/workflows/constraint.yaml new file mode 100644 index 00000000..adcbf004 --- /dev/null +++ b/.github/workflows/constraint.yaml @@ -0,0 +1,41 @@ +# Reference from: +# https://github.com/c-bata/go-prompt/blob/master/.github/workflows/test.yml +name: Constraints +on: + pull_request: + types: [opened, edited, synchronize, reopened] +jobs: + # Lints Pull Request commits with commitlint. + # + # Rules can be referenced: + # https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional + CommitLint: + name: Commit Lint + runs-on: ubuntu-latest + if: contains(fromJSON('["pull_request"]'), github.event_name) + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: wagoid/commitlint-github-action@v5 + + # Lints Pull Request title, because the title will be used as the + # commit message in branch main. + # + # Configuration detail can be referenced: + # https://github.com/marketplace/actions/pull-request-title-rules + PullRequestTitleLint: + name: Pull Request Title Lint + runs-on: ubuntu-latest + if: contains(fromJSON('["pull_request"]'), github.event_name) + steps: + - uses: deepakputhraya/action-pr-title@master + with: + allowed_prefixes: 'build,chore,ci,docs,feat,fix,perf,refactor,revert,style,test' # title should start with the given prefix + disallowed_prefixes: 'WIP,[WIP]' # title should not start with the given prefix + prefix_case_sensitive: false # title prefix are case insensitive + min_length: 5 # Min length of the title + max_length: 80 # Max length of the title + github_token: ${{ github.token }} # Default: ${{ github.token }} + diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b9bc595c..51fd2678 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go 1.19 @@ -33,12 +33,13 @@ jobs: uses: shogo82148/actions-goveralls@v1 with: path-to-profile: coverage.out - Lint: - name: Lint checks + + GolangLint: + name: Golang Lint runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go 1.19 @@ -49,4 +50,4 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.52.2 + version: v1.52.2 \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 57cb344a..0859afe7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -48,7 +48,7 @@ linters: linters-settings: gofumpt: # Select the Go version to target. The default is `1.15`. - lang-version: "1.16" + lang-version: "1.19" # Choose whether or not to use the extra rules that are disabled # by default extra-rules: false @@ -60,4 +60,4 @@ issues: - "ifElseChain: rewrite if-else to switch statement" - "S1000: should use for range instead of for { select {} }" - "SA4004: the surrounding loop is unconditionally terminated" - - "copylocks: call of c\\.Post copies lock value: kcl-lang\\.io/kcl-go/pkg/spec/gpyrpc\\.Ping_Args contains google\\.golang\\.org/protobuf/internal/impl\\.MessageState contains sync\\.Mutex" \ No newline at end of file + - "copylocks: call of c\\.Post copies lock value: kcl-lang\\.io/kcl-go/pkg/spec/gpyrpc\\.Ping_Args contains google\\.golang\\.org/protobuf/internal/impl\\.MessageState contains sync\\.Mutex" diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 00000000..bb8692b4 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,21 @@ +/* + * Copyright The Karbour Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file is the configuration file of [commitlint](https://commitlint.js.org/#/). +// +// Rules can be referenced: +// https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional +module.exports = {extends: ['@commitlint/config-conventional']} diff --git a/pkg/cmd/preview/options_test.go b/pkg/cmd/preview/options_test.go index 877b94ba..b265abd5 100644 --- a/pkg/cmd/preview/options_test.go +++ b/pkg/cmd/preview/options_test.go @@ -6,9 +6,8 @@ import ( "path/filepath" "testing" - "github.com/stretchr/testify/assert" - "github.com/bytedance/mockey" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" compilecmd "kusionstack.io/kusion/pkg/cmd/compile" diff --git a/pkg/cmd/spec/generator.go b/pkg/cmd/spec/generator.go index 29f1dc9c..85ac63a0 100644 --- a/pkg/cmd/spec/generator.go +++ b/pkg/cmd/spec/generator.go @@ -9,12 +9,14 @@ import ( "github.com/acarl005/stripansi" "github.com/pterm/pterm" - "gopkg.in/yaml.v3" "kusionstack.io/kusion/pkg/generator" + "kusionstack.io/kusion/pkg/generator/appconfiguration" "kusionstack.io/kusion/pkg/generator/kcl" + "kusionstack.io/kusion/pkg/log" "kusionstack.io/kusion/pkg/models" + appconfigmodel "kusionstack.io/kusion/pkg/models/appconfiguration" "kusionstack.io/kusion/pkg/projectstack" "kusionstack.io/kusion/pkg/util/pretty" ) @@ -66,18 +68,50 @@ func GenerateSpec(o *generator.Options, project *projectstack.Project, stack *pr switch gt { case projectstack.KCLGenerator: g = &kcl.Generator{} + case projectstack.AppConfigurationGenerator: + appConfig, err := buildAppConfig(o, stack) + if err != nil { + return nil, err + } + g = &appconfiguration.Generator{AppConfiguration: appConfig} default: return nil, fmt.Errorf("unknow generator type:%s", gt) } } - spec, err := g.GenerateSpec(o, stack) + spec, err := g.GenerateSpec(o, project, stack) if err != nil { return nil, errors.New(stripansi.Strip(err.Error())) } return spec, nil } +func buildAppConfig(o *generator.Options, stack *projectstack.Stack) (*appconfigmodel.AppConfiguration, error) { + compileResult, err := kcl.Run(o, stack) + if err != nil { + return nil, err + } + + documents := compileResult.Documents + if len(documents) != 1 { + return nil, fmt.Errorf("invalide more than one AppConfiguration are found in the compile result") + } + + out, err := yaml.Marshal(documents[0]) + if err != nil { + return nil, err + } + + log.Debugf("unmarshal %s to app config", out) + appConfig := &appconfigmodel.AppConfiguration{} + err = yaml.Unmarshal(out, appConfig) + if err != nil { + return nil, err + } + + return appConfig, nil +} + func GenerateSpecFromFile(filePath string) (*models.Spec, error) { b, err := os.ReadFile(filePath) if err != nil { diff --git a/pkg/generator/appconfiguration/app_configuration_generator.go b/pkg/generator/appconfiguration/app_configuration_generator.go new file mode 100644 index 00000000..60430991 --- /dev/null +++ b/pkg/generator/appconfiguration/app_configuration_generator.go @@ -0,0 +1,33 @@ +package appconfiguration + +import ( + "kusionstack.io/kusion/pkg/generator" + "kusionstack.io/kusion/pkg/generator/appconfiguration/generators" + "kusionstack.io/kusion/pkg/models" + "kusionstack.io/kusion/pkg/models/appconfiguration" + "kusionstack.io/kusion/pkg/projectstack" +) + +type Generator struct { + *appconfiguration.AppConfiguration +} + +func (acg *Generator) GenerateSpec( + o *generator.Options, + project *projectstack.Project, + stack *projectstack.Stack, +) (*models.Spec, error) { + spec := &models.Spec{ + Resources: []models.Resource{}, + } + + g, err := generators.NewAppConfigurationGenerator(project.Name, acg.AppConfiguration) + if err != nil { + return nil, err + } + if err = g.Generate(spec); err != nil { + return nil, err + } + + return spec, nil +} diff --git a/pkg/generator/appconfiguration/generators/app_configuration_generator.go b/pkg/generator/appconfiguration/generators/app_configuration_generator.go new file mode 100644 index 00000000..9b0491e7 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/app_configuration_generator.go @@ -0,0 +1,45 @@ +package generators + +import ( + "fmt" + + "kusionstack.io/kusion/pkg/models" + "kusionstack.io/kusion/pkg/models/appconfiguration" +) + +type appConfigurationGenerator struct { + projectName string + ac *appconfiguration.AppConfiguration +} + +func NewAppConfigurationGenerator(projectName string, ac *appconfiguration.AppConfiguration) (Generator, error) { + if len(projectName) == 0 { + return nil, fmt.Errorf("project name must not be empty") + } + + if ac == nil { + return nil, fmt.Errorf("can not find app configuration when generating the Spec") + } + + return &appConfigurationGenerator{ + projectName: projectName, + ac: ac, + }, nil +} + +func (g *appConfigurationGenerator) Generate(spec *models.Spec) error { + if spec.Resources == nil { + spec.Resources = make(models.Resources, 0) + } + + gfs := []NewGeneratorFunc{ + NewNamespaceGeneratorFunc(g.projectName), + NewComponentsGeneratorFunc(g.projectName, g.ac.Components), + } + + if err := callGenerators(spec, gfs...); err != nil { + return err + } + + return nil +} diff --git a/pkg/generator/appconfiguration/generators/components_generator.go b/pkg/generator/appconfiguration/generators/components_generator.go new file mode 100644 index 00000000..dda39748 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/components_generator.go @@ -0,0 +1,55 @@ +package generators + +import ( + "fmt" + + "kusionstack.io/kusion/pkg/models" + "kusionstack.io/kusion/pkg/models/appconfiguration/component" +) + +type componentsGenerator struct { + projectName string + components map[string]component.Component +} + +func NewComponentsGenerator(projectName string, components map[string]component.Component) (Generator, error) { + if len(projectName) == 0 { + return nil, fmt.Errorf("project name must not be empty") + } + + return &componentsGenerator{ + projectName: projectName, + components: components, + }, nil +} + +func NewComponentsGeneratorFunc(projectName string, components map[string]component.Component) NewGeneratorFunc { + return func() (Generator, error) { + return NewComponentsGenerator(projectName, components) + } +} + +func (g *componentsGenerator) Generate(spec *models.Spec) error { + if spec.Resources == nil { + spec.Resources = make(models.Resources, 0) + } + + if g.components != nil { + if err := foreachOrderedComponents(g.components, func(compName string, comp component.Component) error { + gfs := []NewGeneratorFunc{ + NewDeploymentGeneratorFunc(g.projectName, compName, &comp), + NewJobGeneratorFunc(g.projectName, compName, &comp), + } + + if err := callGenerators(spec, gfs...); err != nil { + return err + } + + return nil + }); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/generator/appconfiguration/generators/deployment_generator.go b/pkg/generator/appconfiguration/generators/deployment_generator.go new file mode 100644 index 00000000..e47ce920 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/deployment_generator.go @@ -0,0 +1,112 @@ +package generators + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "kusionstack.io/kusion/pkg/models" + "kusionstack.io/kusion/pkg/models/appconfiguration/component" +) + +// deploymentGenerator is a struct for generating Deployment +// resources. +type deploymentGenerator struct { + projectName string + compName string + comp *component.Component +} + +// NewDeploymentGenerator returns a new deploymentGenerator instance. +func NewDeploymentGenerator( + projectName string, + compName string, + comp *component.Component, +) (Generator, error) { + if len(projectName) == 0 { + return nil, fmt.Errorf("project name must not be empty") + } + + if len(compName) == 0 { + return nil, fmt.Errorf("component name must not be empty") + } + + if comp == nil { + return nil, fmt.Errorf("component must not be nil") + } + + return &deploymentGenerator{ + projectName: projectName, + compName: compName, + comp: comp, + }, nil +} + +// NewDeploymentGeneratorFunc returns a new NewGeneratorFunc that +// returns a deploymentGenerator instance. +func NewDeploymentGeneratorFunc( + projectName string, + compName string, + comp *component.Component, +) NewGeneratorFunc { + return func() (Generator, error) { + return NewDeploymentGenerator(projectName, compName, comp) + } +} + +// Generate generates a Deployment resource to the given spec. +func (g *deploymentGenerator) Generate(spec *models.Spec) error { + lrs := g.comp.LongRunningService + if lrs == nil { + return nil + } + + // Create an empty resource slice if it doesn't exist yet. + if spec.Resources == nil { + spec.Resources = make(models.Resources, 0) + } + + // Create a slice of containers based on the component's + // containers. + containers, err := toOrderedContainers(lrs.Containers) + if err != nil { + return err + } + + // Create a Deployment object based on the component's + // configuration. + resource := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Labels: uniqueComponentLabels(g.projectName, g.compName), + Name: uniqueComponentName(g.projectName, g.compName), + Namespace: g.projectName, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(int32(lrs.Replicas)), + Selector: &metav1.LabelSelector{ + MatchLabels: uniqueComponentLabels(g.projectName, g.compName), + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: uniqueComponentLabels(g.projectName, g.compName), + }, + Spec: v1.PodSpec{ + Containers: containers, + }, + }, + }, + } + + // Add the Deployment resource to the spec. + return appendToSpec( + kubernetesResourceID(resource.TypeMeta, resource.ObjectMeta), + resource, + spec, + ) +} diff --git a/pkg/generator/appconfiguration/generators/job_generator.go b/pkg/generator/appconfiguration/generators/job_generator.go new file mode 100644 index 00000000..43e28399 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/job_generator.go @@ -0,0 +1,98 @@ +package generators + +import ( + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "kusionstack.io/kusion/pkg/models" + "kusionstack.io/kusion/pkg/models/appconfiguration/component" +) + +type jobGenerator struct { + projectName string + compName string + comp *component.Component +} + +func NewJobGenerator(projectName, compName string, comp *component.Component) (Generator, error) { + return &jobGenerator{ + projectName: projectName, + compName: compName, + comp: comp, + }, nil +} + +func NewJobGeneratorFunc(projectName, compName string, comp *component.Component) NewGeneratorFunc { + return func() (Generator, error) { + return NewJobGenerator(projectName, compName, comp) + } +} + +func (g *jobGenerator) Generate(spec *models.Spec) error { + job := g.comp.Job + if job == nil { + return nil + } + + if spec.Resources == nil { + spec.Resources = make(models.Resources, 0) + } + + meta := metav1.ObjectMeta{ + Namespace: g.projectName, + Name: uniqueComponentName(g.projectName, g.compName), + Labels: g.comp.Labels, + Annotations: g.comp.Annotations, + } + + containers, err := toOrderedContainers(job.Containers) + if err != nil { + return err + } + jobSpec := batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: uniqueComponentLabels(g.projectName, g.compName), + }, + Spec: corev1.PodSpec{ + Containers: containers, + }, + }, + } + + if job.Schedule == "" { + resource := &batchv1.Job{ + ObjectMeta: meta, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: batchv1.SchemeGroupVersion.String(), + }, + Spec: jobSpec, + } + return appendToSpec( + kubernetesResourceID(resource.TypeMeta, resource.ObjectMeta), + resource, + spec, + ) + } + + resource := &batchv1.CronJob{ + ObjectMeta: meta, + TypeMeta: metav1.TypeMeta{ + Kind: "CronJob", + APIVersion: batchv1.SchemeGroupVersion.String(), + }, + Spec: batchv1.CronJobSpec{ + JobTemplate: batchv1.JobTemplateSpec{ + Spec: jobSpec, + }, + Schedule: job.Schedule, + }, + } + return appendToSpec( + kubernetesResourceID(resource.TypeMeta, resource.ObjectMeta), + resource, + spec, + ) +} diff --git a/pkg/generator/appconfiguration/generators/job_generator_test.go b/pkg/generator/appconfiguration/generators/job_generator_test.go new file mode 100644 index 00000000..64040227 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/job_generator_test.go @@ -0,0 +1,163 @@ +package generators + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "kusionstack.io/kusion/pkg/models" + "kusionstack.io/kusion/pkg/models/appconfiguration/component" + "kusionstack.io/kusion/pkg/models/appconfiguration/component/container" + "kusionstack.io/kusion/pkg/models/appconfiguration/component/workload" +) + +func Test_jobGenerator_Generate(t *testing.T) { + type fields struct { + projectName string + compName string + comp *component.Component + } + tests := []struct { + name string + fields fields + wantErr bool + in *models.Spec + want *models.Spec + }{ + { + name: "test1", + fields: fields{ + projectName: "proj1", + compName: "comp1", + comp: &component.Component{ + Job: &workload.Job{ + Containers: map[string]container.Container{ + "container1": { + Image: "nginx:v1", + }, + }, + }, + }, + }, + in: &models.Spec{}, + want: &models.Spec{ + Resources: models.Resources{ + { + ID: "batch/v1:Job:proj1:proj1-comp1", + Type: "Kubernetes", + Attributes: map[string]interface{}{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": map[string]interface{}{ + "creationTimestamp": nil, + "name": "proj1-comp1", + "namespace": "proj1", + }, + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": nil, + "labels": map[string]interface{}{ + "app.kubernetes.io/component": "comp1", + "app.kubernetes.io/name": "proj1", + }, + }, + "spec": map[string]interface{}{ + "containers": []interface{}{map[string]interface{}{ + "image": "nginx:v1", + "name": "container1", + "resources": map[string]interface{}{}, + }}, + }, + }, + }, + "status": map[string]interface{}{}, + }, + DependsOn: nil, + Extensions: nil, + }, + }, + }, + }, + { + name: "test2", + fields: fields{ + projectName: "proj2", + compName: "comp2", + comp: &component.Component{ + Job: &workload.Job{ + Containers: map[string]container.Container{ + "container2": { + Image: "nginx:v1", + }, + }, + Schedule: "* * * * *", + }, + }, + }, + in: &models.Spec{}, + want: &models.Spec{ + Resources: models.Resources{ + { + ID: "batch/v1:CronJob:proj2:proj2-comp2", + Type: "Kubernetes", + Attributes: map[string]interface{}{ + "apiVersion": "batch/v1", + "kind": "CronJob", + "metadata": map[string]interface{}{ + "creationTimestamp": nil, + "name": "proj2-comp2", + "namespace": "proj2", + }, + "spec": map[string]interface{}{ + "jobTemplate": map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": nil, + }, + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": nil, + "labels": map[string]interface{}{ + "app.kubernetes.io/component": "comp2", + "app.kubernetes.io/name": "proj2", + }, + }, + "spec": map[string]interface{}{ + "containers": []interface{}{map[string]interface{}{ + "image": "nginx:v1", + "name": "container2", + "resources": map[string]interface{}{}, + }}, + }, + }, + }, + }, + "schedule": "* * * * *", + }, + "status": map[string]interface{}{}, + }, + DependsOn: nil, + Extensions: nil, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := &jobGenerator{ + projectName: tt.fields.projectName, + compName: tt.fields.compName, + comp: tt.fields.comp, + } + err := g.Generate(tt.in) + if err != nil { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.want, tt.in) + } + }) + } +} diff --git a/pkg/generator/appconfiguration/generators/namespace_generator.go b/pkg/generator/appconfiguration/generators/namespace_generator.go new file mode 100644 index 00000000..93a60ac4 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/namespace_generator.go @@ -0,0 +1,50 @@ +package generators + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "kusionstack.io/kusion/pkg/models" +) + +type namespaceGenerator struct { + projectName string +} + +func NewNamespaceGenerator(projectName string) (Generator, error) { + if len(projectName) == 0 { + return nil, fmt.Errorf("project name must not be empty") + } + + return &namespaceGenerator{ + projectName: projectName, + }, nil +} + +func NewNamespaceGeneratorFunc(projectName string) NewGeneratorFunc { + return func() (Generator, error) { + return NewNamespaceGenerator(projectName) + } +} + +func (g *namespaceGenerator) Generate(spec *models.Spec) error { + if spec.Resources == nil { + spec.Resources = make(models.Resources, 0) + } + + ns := &v1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{Name: g.projectName}, + } + + return appendToSpec( + kubernetesResourceID(ns.TypeMeta, ns.ObjectMeta), + ns, + spec, + ) +} diff --git a/pkg/generator/appconfiguration/generators/types.go b/pkg/generator/appconfiguration/generators/types.go new file mode 100644 index 00000000..277a06e6 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/types.go @@ -0,0 +1,9 @@ +package generators + +import "kusionstack.io/kusion/pkg/models" + +type Generator interface { + Generate(spec *models.Spec) error +} + +type NewGeneratorFunc func() (Generator, error) diff --git a/pkg/generator/appconfiguration/generators/util.go b/pkg/generator/appconfiguration/generators/util.go new file mode 100644 index 00000000..d75283f0 --- /dev/null +++ b/pkg/generator/appconfiguration/generators/util.go @@ -0,0 +1,171 @@ +package generators + +import ( + "sort" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "kusionstack.io/kusion/pkg/generator" + "kusionstack.io/kusion/pkg/models" + "kusionstack.io/kusion/pkg/models/appconfiguration/component" + "kusionstack.io/kusion/pkg/models/appconfiguration/component/container" +) + +// kubernetesResourceID returns the unique ID of a Kubernetes resource +// based on its type and metadata. +func kubernetesResourceID(typeMeta metav1.TypeMeta, objectMeta metav1.ObjectMeta) string { + // resource id example: apps/v1:Deployment:code-city:code-citydev + id := typeMeta.APIVersion + ":" + typeMeta.Kind + ":" + if objectMeta.Namespace != "" { + id += objectMeta.Namespace + ":" + } + id += objectMeta.Name + return id +} + +// callGeneratorFuncs calls each NewGeneratorFunc in the given slice +// and returns a slice of Generator instances. +func callGeneratorFuncs(newGenerators ...NewGeneratorFunc) ([]Generator, error) { + gs := make([]Generator, 0, len(newGenerators)) + for _, newGenerator := range newGenerators { + if g, err := newGenerator(); err != nil { + return nil, err + } else { + gs = append(gs, g) + } + } + return gs, nil +} + +// callGenerators calls the Generate method of each Generator instance +// returned by the given NewGeneratorFuncs. +func callGenerators(spec *models.Spec, newGenerators ...NewGeneratorFunc) error { + gs, err := callGeneratorFuncs(newGenerators...) + if err != nil { + return err + } + for _, g := range gs { + if err := g.Generate(spec); err != nil { + return err + } + } + return nil +} + +// appendToSpec adds a Kubernetes resource to a spec's resources +// slice. +func appendToSpec(resourceID string, resource any, spec *models.Spec) error { + unstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(resource) + if err != nil { + return err + } + r := models.Resource{ + ID: resourceID, + Type: generator.Kubernetes, + Attributes: unstructured, + DependsOn: nil, + Extensions: nil, + } + spec.Resources = append(spec.Resources, r) + return nil +} + +// uniqueComponentName returns a unique name for a component based on +// its project and name. +func uniqueComponentName(projectName, compName string) string { + return projectName + "-" + compName +} + +// uniqueComponentLabels returns a map of labels that identify a +// component based on its project and name. +func uniqueComponentLabels(projectName, compName string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": projectName, + "app.kubernetes.io/component": compName, + } +} + +// int32Ptr returns a pointer to an int32 value. +func int32Ptr(i int32) *int32 { + return &i +} + +// foreachOrderedContainers executes the given function on each +// container in the map in order of their keys. +func foreachOrderedContainers( + m map[string]container.Container, + f func(containerName string, c container.Container) error, +) error { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + + sort.Strings(keys) + + for _, k := range keys { + v := m[k] + if err := f(k, v); err != nil { + return err + } + } + + return nil +} + +func toOrderedContainers(appContainers map[string]container.Container) ([]corev1.Container, error) { + // Create a slice of containers based on the component's + // containers. + containers := []corev1.Container{} + if err := foreachOrderedContainers(appContainers, func(containerName string, c container.Container) error { + // Create a slice of env vars based on the container's + // envvars. + envs := []corev1.EnvVar{} + for k, v := range c.Env { + envs = append(envs, corev1.EnvVar{ + Name: k, + Value: v, + }) + } + + // Create a container object and append it to the containers + // slice. + containers = append(containers, corev1.Container{ + Name: containerName, + Image: c.Image, + Command: c.Command, + Args: c.Args, + WorkingDir: c.WorkingDir, + Env: envs, + }) + return nil + }); err != nil { + return nil, err + } + return containers, nil +} + +// foreachOrderedComponents executes the given function on each +// component in the map in order of their keys. +func foreachOrderedComponents( + m map[string]component.Component, + f func(compName string, comp component.Component) error, +) error { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + + sort.Strings(keys) + + for _, k := range keys { + v := m[k] + if err := f(k, v); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index f7c5f0df..1226a06e 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -8,7 +8,7 @@ import ( // Generator represents a way to generate Spec. Usually, it is implemented by KCL, but we make it as an interface for a more general usage. // Anyone who implements this interface is regarded as a Generator, and can be integrated by the Kusion workflow. type Generator interface { - GenerateSpec(o *Options, stack *projectstack.Stack) (*models.Spec, error) + GenerateSpec(o *Options, project *projectstack.Project, stack *projectstack.Stack) (*models.Spec, error) } const ( diff --git a/pkg/generator/kcl/kcl_generator.go b/pkg/generator/kcl/kcl_generator.go index bdbaea6d..887a1c77 100644 --- a/pkg/generator/kcl/kcl_generator.go +++ b/pkg/generator/kcl/kcl_generator.go @@ -45,7 +45,7 @@ func EnableRPC() bool { return !enableRest } -func (g *Generator) GenerateSpec(o *generator.Options, stack *projectstack.Stack) (*models.Spec, error) { +func (g *Generator) GenerateSpec(o *generator.Options, _ *projectstack.Project, stack *projectstack.Stack) (*models.Spec, error) { compileResult, err := Run(o, stack) if err != nil { return nil, err diff --git a/pkg/generator/kcl/kcl_generator_test.go b/pkg/generator/kcl/kcl_generator_test.go index dabfef1b..b532f127 100644 --- a/pkg/generator/kcl/kcl_generator_test.go +++ b/pkg/generator/kcl/kcl_generator_test.go @@ -87,16 +87,15 @@ func TestGenerateSpec(t *testing.T) { } g := &Generator{} - got, err := g.GenerateSpec( - &generator.Options{ - WorkDir: tt.args.workDir, - Filenames: tt.args.filenames, - Settings: tt.args.settings, - Arguments: tt.args.arguments, - Overrides: tt.args.overrides, - DisableNone: tt.args.disableNone, - OverrideAST: tt.args.overrideAST, - }, fakeStack) + got, err := g.GenerateSpec(&generator.Options{ + WorkDir: tt.args.workDir, + Filenames: tt.args.filenames, + Settings: tt.args.settings, + Arguments: tt.args.arguments, + Overrides: tt.args.overrides, + DisableNone: tt.args.disableNone, + OverrideAST: tt.args.overrideAST, + }, nil, fakeStack) if (err != nil) != tt.wantErr { t.Errorf("Compile() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/models/appconfiguration/appconfiguration.go b/pkg/models/appconfiguration/appconfiguration.go new file mode 100644 index 00000000..5cb20de5 --- /dev/null +++ b/pkg/models/appconfiguration/appconfiguration.go @@ -0,0 +1,33 @@ +package appconfiguration + +import "kusionstack.io/kusion/pkg/models/appconfiguration/component" + +// AppConfiguration is a developer-centric definition that describes how to run an Application. +// This application model builds upon a decade of experience at AntGroup running super large scale +// internal developer platform, combined with best-of-breed ideas and practices from the community. +// +// Example +// +// components: +// proxy: +// containers: +// nginx: +// image: nginx:v1 +// command: +// - /bin/sh +// - -c +// - echo hi +// args: +// - /bin/sh +// - -c +// - echo hi +// env: +// env1: VALUE +// env2: secret://sec-name/key +// workingDir: /tmp +// replicas: 2 +type AppConfiguration struct { + // Component defines the delivery artifact of one application. + // Each application can be composed by multiple components. + Components map[string]component.Component `json:"components,omitempty" yaml:"components,omitempty"` +} diff --git a/pkg/models/appconfiguration/component/component.go b/pkg/models/appconfiguration/component/component.go new file mode 100644 index 00000000..4b515ae5 --- /dev/null +++ b/pkg/models/appconfiguration/component/component.go @@ -0,0 +1,22 @@ +package component + +import ( + "kusionstack.io/kusion/pkg/models/appconfiguration/component/workload" +) + +type Component struct { + Job *workload.Job + LongRunningService *workload.LongRunningService + + // List of Workload supporting accessory. Accessory defines various runtime capabilities and operation functionalities. + + // Variables for Day-2 Operation. + + // Variables for Workload scheduling. + + // Other metadata info + + // Labels and annotations can be used to attach arbitrary metadata as key-value pairs to resources. + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` + Annotations map[string]string `yaml:"annotations,omitempty" json:"annotations,omitempty"` +} diff --git a/pkg/models/appconfiguration/component/container/container.go b/pkg/models/appconfiguration/component/container/container.go new file mode 100644 index 00000000..8762d497 --- /dev/null +++ b/pkg/models/appconfiguration/component/container/container.go @@ -0,0 +1,17 @@ +package container + +type Container struct { + // Image to run for this container + Image string `yaml:"image,omitempty" json:"image,omitempty"` + // Entrypoint array. + // The image's ENTRYPOINT is used if this is not provided. + Command []string `yaml:"command,omitempty" json:"command,omitempty"` + // Arguments to the entrypoint. + // The image's CMD is used if this is not provided. + Args []string `yaml:"args,omitempty" json:"args,omitempty"` + // Collection of environment variables to set in the container. + // The value of environment variable may be static text or a value from a secret. + Env map[string]string `yaml:"env,omitempty" json:"env,omitempty"` + // The current working directory of the running process defined in entrypoint. + WorkingDir string `yaml:"workingDir,omitempty" json:"workingDir,omitempty"` +} diff --git a/pkg/models/appconfiguration/component/workload/job.go b/pkg/models/appconfiguration/component/workload/job.go new file mode 100644 index 00000000..6d24bb15 --- /dev/null +++ b/pkg/models/appconfiguration/component/workload/job.go @@ -0,0 +1,12 @@ +package workload + +import "kusionstack.io/kusion/pkg/models/appconfiguration/component/container" + +type Job struct { + // The templates of containers to be ran. + Containers map[string]container.Container `yaml:"containers,omitempty" json:"containers,omitempty"` + + // The schedule in Cron format + // Only supported when workloadType is Job. + Schedule string +} diff --git a/pkg/models/appconfiguration/component/workload/long_running_service.go b/pkg/models/appconfiguration/component/workload/long_running_service.go new file mode 100644 index 00000000..3b2108e9 --- /dev/null +++ b/pkg/models/appconfiguration/component/workload/long_running_service.go @@ -0,0 +1,13 @@ +package workload + +import "kusionstack.io/kusion/pkg/models/appconfiguration/component/container" + +type LongRunningService struct { + // The templates of containers to be ran. + Containers map[string]container.Container `yaml:"containers,omitempty" json:"containers,omitempty"` + + // The number of containers that should be ran. + // Default is 2 to meet high availability requirements. + // Only supported when workloadType is LongRunningService + Replicas int `yaml:"replicas,omitempty" json:"replicas,omitempty"` +} diff --git a/pkg/projectstack/types.go b/pkg/projectstack/types.go index fcf18a46..0972e018 100644 --- a/pkg/projectstack/types.go +++ b/pkg/projectstack/types.go @@ -19,13 +19,14 @@ var ( ) const ( - StackFile = "stack.yaml" - ProjectFile = "project.yaml" - CiTestDir = "ci-test" - SettingsFile = "settings.yaml" - StdoutGoldenFile = "stdout.golden.yaml" - KclFile = "kcl.yaml" - KCLGenerator GeneratorType = "KCL" + StackFile = "stack.yaml" + ProjectFile = "project.yaml" + CiTestDir = "ci-test" + SettingsFile = "settings.yaml" + StdoutGoldenFile = "stdout.golden.yaml" + KclFile = "kcl.yaml" + KCLGenerator GeneratorType = "KCL" + AppConfigurationGenerator GeneratorType = "AppConfiguration" ) type GeneratorType string