Skip to content

Commit

Permalink
feat: save attributes into manifest
Browse files Browse the repository at this point in the history 
Signed-off-by: Artur Troian <[email protected]>
  • Loading branch information
@troian
troian committed Sep 1, 2023
1 parent 7432e8b commit 907e061
Show file tree
Hide file tree
Showing 23 changed files with 360 additions and 202 deletions.
3 changes: 2 additions & 1 deletion _run/common-kube.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ kube-upload-images-default:
kube-upload-crd:
$(SETUP_KUBE) --crd=$(CRD_FILE) $(KUBE_SSH_NODE_NAME) init

$(KUBE_CREATE): $(AP_RUN_DIR) kube-cluster-create-$(KUBE_CLUSTER_CREATE_TARGET) kube-upload-crd
$(KUBE_CREATE): $(AP_RUN_DIR) kube-cluster-create-$(KUBE_CLUSTER_CREATE_TARGET)
$(SETUP_KUBE) --crd=$(CRD_FILE) $(KUBE_SSH_NODE_NAME) init
touch $@

.INTERMEDIATE: kube-cluster-create-default
Expand Down
6 changes: 3 additions & 3 deletions cluster/kube/apply.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,22 +111,22 @@ func applyDeployment(ctx context.Context, kc kubernetes.Interface, b builder.Dep

func applyStatefulSet(ctx context.Context, kc kubernetes.Interface, b builder.StatefulSet) error {
obj, err := kc.AppsV1().StatefulSets(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "deployments-get", err, errors.IsNotFound)
metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "statefulset-get", err, errors.IsNotFound)

switch {
case err == nil:
obj, err = b.Update(obj)

if err == nil {
_, err = kc.AppsV1().StatefulSets(b.NS()).Update(ctx, obj, metav1.UpdateOptions{})
metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "deployments-update", err)
metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "statefulset-update", err)

}
case errors.IsNotFound(err):
obj, err = b.Create()
if err == nil {
_, err = kc.AppsV1().StatefulSets(b.NS()).Create(ctx, obj, metav1.CreateOptions{})
metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "deployments-create", err)
metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "statefulset-create", err)
}
}
return err
Expand Down
5 changes: 2 additions & 3 deletions cluster/kube/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
corev1 "k8s.io/api/core/v1"
eventsv1 "k8s.io/api/events/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
kubeErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apimachinery/pkg/watch"
Expand Down Expand Up @@ -159,7 +158,7 @@ func (c *client) GetManifestGroup(ctx context.Context, lID mtypes.LeaseID) (bool
})

if err != nil {
if kubeErrors.IsNotFound(err) {
if kerrors.IsNotFound(err) {
c.log.Info("CRD manifest not found", "lease-ns", leaseNamespace)
return false, crd.ManifestGroup{}, nil
}
Expand Down Expand Up @@ -764,7 +763,7 @@ func (c *client) leaseExists(ctx context.Context, lid mtypes.LeaseID) error {
})

if err != nil {
if kubeErrors.IsNotFound(err) {
if kerrors.IsNotFound(err) {
return kubeclienterrors.ErrLeaseNotFound
}

Expand Down
1 change: 0 additions & 1 deletion cluster/kube/client_hostname_connections.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ func (c *client) DeclareHostname(ctx context.Context, lID mtypes.LeaseID, host s
ServiceName: serviceName,
ExternalPort: externalPort,
},
Status: crd.ProviderHostStatus{},
}

c.log.Info("declaring hostname", "lease", lID, "service-name", serviceName, "external-port", externalPort, "host", host)
Expand Down
1 change: 0 additions & 1 deletion cluster/kube/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,6 @@ func fakeProviderHost(hostname string, leaseID mtypes.LeaseID, serviceName strin
ServiceName: serviceName,
ExternalPort: externalPort,
},
Status: crd.ProviderHostStatus{},
}
}

Expand Down
1 change: 0 additions & 1 deletion cluster/kube/cluster_ip_connections.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@ func (c *client) DeclareIP(ctx context.Context, lID mtypes.LeaseID, serviceName
Protocol: proto.ToString(),
Port: port,
},
Status: akashtypes.ProviderLeasedIPStatus{},
}

c.log.Info("declaring leased ip", "lease", lID,
Expand Down
12 changes: 8 additions & 4 deletions gateway/rest/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,15 @@ func mockManifestGroups(m integrationMocks, leaseID mtypes.LeaseID) {
Args: nil,
Env: nil,
Resources: v2beta2.Resources{
CPU: 1000,
Memory: "3333",
Storage: []v2beta2.ResourcesStorage{
CPU: v2beta2.ResourceCPU{
Units: 1000,
},
Memory: v2beta2.ResourceMemory{
Size: "3333",
},
Storage: v2beta2.ResourceStorage{
{
Name: "",
Name: "default",
Size: "4444",
},
},
Expand Down
12 changes: 8 additions & 4 deletions gateway/rest/router_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,11 +628,15 @@ func mockManifestGroupsForRouterTest(rt *routerTest, leaseID mtypes.LeaseID) {
Args: nil,
Env: nil,
Resources: v2beta2.Resources{
CPU: 1000,
Memory: "3333",
Storage: []v2beta2.ResourcesStorage{
CPU: v2beta2.ResourceCPU{
Units: 1000,
},
Memory: v2beta2.ResourceMemory{
Size: "3333",
},
Storage: v2beta2.ResourceStorage{
{
Name: "",
Name: "default",
Size: "4444",
},
},
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.21
toolchain go1.21.0

require (
github.com/akash-network/akash-api v0.0.28
github.com/akash-network/akash-api v0.0.29
github.com/akash-network/node v0.24.2
github.com/avast/retry-go/v4 v4.5.0
github.com/boz/go-lifecycle v0.1.1
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,8 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/akash-network/akash-api v0.0.28 h1:QEoQJI02NqJgxNPH4C10XSEB7VWVnEh8nUVML7wdqME=
github.com/akash-network/akash-api v0.0.28/go.mod h1:9aIEhpvAyGQJ4/p1cPo18lG67S0QGT//JLueAXqk6Kw=
github.com/akash-network/akash-api v0.0.29 h1:S+GiYldK1UlpfHsSKOccQWGPsW3xqztwi5745lumgv4=
github.com/akash-network/akash-api v0.0.29/go.mod h1:9aIEhpvAyGQJ4/p1cPo18lG67S0QGT//JLueAXqk6Kw=
github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0=
github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo=
Expand Down
67 changes: 58 additions & 9 deletions pkg/apis/akash.network/crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,24 +72,73 @@ spec:
type: number
format: uint32
cpu:
type: number
format: uint32
memory:
type: string
format: uint64
type: object
properties:
units:
type: number
format: uint32
attributes:
type: array
nullable: true
items:
type: object
properties:
key:
type: string
value:
type: string
gpu:
type: number
format: uint32
type: object
properties:
units:
type: number
format: uint32
attributes:
type: array
nullable: true
items:
type: object
properties:
key:
type: string
value:
type: string
memory:
type: object
properties:
size:
type: string
format: uint64
attributes:
type: array
nullable: true
items:
type: object
properties:
key:
type: string
value:
type: string
storage:
type: array
items:
type: object
properties:
name:
type: string
size:
type: string
format: uint64
name:
type: string
attributes:
type: array
nullable: true
items:
type: object
properties:
key:
type: string
value:
type: string
count:
type: number
format: uint64
Expand Down
17 changes: 7 additions & 10 deletions pkg/apis/akash.network/v2beta2/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ type Manifest struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata"`

Spec ManifestSpec `json:"spec,omitempty"`
Status ManifestStatus `json:"status,omitempty"`
Spec ManifestSpec `json:"spec,omitempty"`
}

// ManifestList stores metadata and items list of manifest
Expand All @@ -36,12 +35,10 @@ type ManifestService struct {
// Service name
Name string `json:"name,omitempty"`
// Docker image
Image string `json:"image,omitempty"`
Command []string `json:"command,omitempty"`
Args []string `json:"args,omitempty"`
Env []string `json:"env,omitempty"`
// Resource requirements
// in current version of CRD it is named as unit
Image string `json:"image,omitempty"`
Command []string `json:"command,omitempty"`
Args []string `json:"args,omitempty"`
Env []string `json:"env,omitempty"`
Resources Resources `json:"resources"`
// Number of instances
Count uint32 `json:"count,omitempty"`
Expand Down Expand Up @@ -218,7 +215,7 @@ func manifestGroupToCRD(m *mani.Group, settings ClusterSettings) (ManifestGroup,
}

func (ms *ManifestService) fromCRD() (mani.Service, error) {
res, err := ms.Resources.fromCRD()
res, err := ms.Resources.ToAkash()
if err != nil {
return mani.Service{}, err
}
Expand Down Expand Up @@ -267,7 +264,7 @@ func (ms *ManifestService) fromCRD() (mani.Service, error) {
}

func manifestServiceFromProvider(ams mani.Service, schedulerParams *SchedulerParams) (ManifestService, error) {
resources, err := resourceUnitsFromAkash(ams.Resources)
resources, err := resourcesFromAkash(ams.Resources)
if err != nil {
return ManifestService{}, err
}
Expand Down
59 changes: 46 additions & 13 deletions pkg/apis/akash.network/v2beta2/migrate/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package migrate

import (
"fmt"
"strconv"

dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3"
"github.com/akash-network/node/sdl"

"github.com/akash-network/provider/pkg/apis/akash.network/v2beta1"
"github.com/akash-network/provider/pkg/apis/akash.network/v2beta2"
Expand Down Expand Up @@ -109,7 +111,7 @@ func ManifestServiceExposeFromV2beta1(from []v2beta1.ManifestServiceExpose) []v2
}

func ManifestResourcesFromV2beta1(res dtypes.ResourceUnits, from v2beta1.ResourceUnits, count uint32) (v2beta2.Resources, error) {
var resID uint32
var gRes *dtypes.ResourceUnit

svcRes, err := from.ToAkash()
if err != nil {
Expand Down Expand Up @@ -139,29 +141,60 @@ func ManifestResourcesFromV2beta1(res dtypes.ResourceUnits, from v2beta1.Resourc
if res[idx].Count >= count {
res[idx].Count -= count

resID = units.ID
gRes = &res[idx]

break
}
}
}

if resID == 0 {
if gRes == nil {
return v2beta2.Resources{}, fmt.Errorf("over-utilized resource group") // nolint goerr113
}

to := v2beta2.Resources{
ID: resID,
CPU: from.CPU,
GPU: 0, // GPU has been introduced in v2beta2
Memory: from.Memory,
Storage: make([]v2beta2.ResourcesStorage, 0, len(from.Storage)),
}
ID: gRes.ID,
CPU: v2beta2.ResourceCPU{
Units: from.CPU,
Attributes: gRes.CPU.Attributes.Dup(),
},
GPU: v2beta2.ResourceGPU{
Units: 0,
Attributes: nil,
},
Memory: v2beta2.ResourceMemory{
Size: strconv.FormatUint(gRes.Memory.Quantity.Value(), 10),
Attributes: gRes.Memory.Attributes.Dup(),
},
Storage: make(v2beta2.ResourceStorage, 0, len(from.Storage)),
}

volumes := make(map[string]string)

for _, vl := range from.Storage {
volumes[vl.Name] = vl.Size
}

for _, storage := range gRes.Storage {
name := storage.Name

attrVal := storage.Attributes.Find(sdl.StorageAttributePersistent)
persistent, _ := attrVal.AsBool()

if persistent {
for nm, size := range volumes {
if nm != "default" && size == strconv.FormatUint(storage.Quantity.Value(), 10) {
name = nm
delete(volumes, nm)
break
}
}
}

for _, storage := range from.Storage {
to.Storage = append(to.Storage, v2beta2.ResourcesStorage{
Name: storage.Name,
Size: storage.Size,
to.Storage = append(to.Storage, v2beta2.ResourceVolume{
Name: name,
Size: strconv.FormatUint(storage.Quantity.Value(), 10),
Attributes: storage.Attributes.Dup(),
})
}

Expand Down
3 changes: 1 addition & 2 deletions pkg/apis/akash.network/v2beta2/provider_host.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ type ProviderHost struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata"`

Spec ProviderHostSpec `json:"spec,omitempty"`
Status ProviderHostStatus `json:"status,omitempty"`
Spec ProviderHostSpec `json:"spec,omitempty"`
}

// ProviderHostList
Expand Down
3 changes: 1 addition & 2 deletions pkg/apis/akash.network/v2beta2/provider_lease_ip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ type ProviderLeasedIP struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata"`

Spec ProviderLeasedIPSpec `json:"spec,omitempty"`
Status ProviderLeasedIPStatus `json:"status,omitempty"`
Spec ProviderLeasedIPSpec `json:"spec,omitempty"`
}

// ProviderLeasedIPList
Expand Down
Loading

0 comments on commit 907e061

Please sign in to comment.