Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Install CNI network plugins only for specific CNIs #17162

Merged
merged 6 commits into from
Jan 3, 2025
Merged

Install CNI network plugins only for specific CNIs #17162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2b604ad
Update CNI network plugins
hakman Jan 2, 2025
bc4eb91
Install CNI network plugins only for specific CNIs
hakman Jan 2, 2025
24283de
Add CNI network plugins hashes
hakman Jan 2, 2025
284e0f7
hack/update-expected.sh
hakman Jan 2, 2025
216af9a
Reuse logic for checking if CNI network plugins need to be installed
hakman Jan 3, 2025
579fb6b
hack/update-expected.sh
hakman Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
37 changes: 37 additions & 0 deletions hack/generate-asset-hashes.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,35 @@ EOF
done
}

function generate_cni_hashes() {
prefix=$1
patch=$2

cat > "${REPO_ROOT}/pkg/assets/assetdata/cni-${prefix}.yaml" <<EOF
# This file is generated by generate-asset-hashes.sh

filestores:
- base: https://storage.googleapis.com/k8s-artifacts-cni/release/

files:
EOF

version="${prefix}.${patch}"
echo "cni ${version}"

# We exclude some files that we don't currently need, to keep the size down
go run ./pkg/assets/assetdata/tools/cmd/generatefileassets \
--base https://storage.googleapis.com/k8s-artifacts-cni/release/ \
--prefix "v${version}/" \
--sums "https://storage.googleapis.com/k8s-artifacts-cni/release/v${version}/cni-plugins-linux-amd64-v${version}.tgz.sha256" \
| sed "s@files:@# cni ${version}@g" >> "${REPO_ROOT}/pkg/assets/assetdata/cni-${prefix}.yaml"
go run ./pkg/assets/assetdata/tools/cmd/generatefileassets \
--base https://storage.googleapis.com/k8s-artifacts-cni/release/ \
--prefix "v${version}/" \
--sums "https://storage.googleapis.com/k8s-artifacts-cni/release/v${version}/cni-plugins-linux-arm64-v${version}.tgz.sha256" \
| sed "s@files:@# cni ${version}@g" >> "${REPO_ROOT}/pkg/assets/assetdata/cni-${prefix}.yaml"
}

# Generate k8s hashes.
# The first argument is the major and minor version, the second is the maximum patch version.
generate_k8s_hashes 1.25 16
Expand All @@ -102,3 +131,11 @@ generate_k8s_hashes 1.31 3
# The first argument is the major and minor version, the second is the maximum patch version.
generate_runc_hashes 1.1 15
generate_runc_hashes 1.2 3

# Generate CNI network plugins hashes.
generate_cni_hashes 0.9 1
generate_cni_hashes 1.2 0
generate_cni_hashes 1.3 0
generate_cni_hashes 1.4 1
generate_cni_hashes 1.5 1
generate_cni_hashes 1.6 1
6 changes: 4 additions & 2 deletions nodeup/pkg/model/networking/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@ var _ fi.NodeupModelBuilder = &CommonBuilder{}

// Build is responsible for copying the common CNI binaries
func (b *CommonBuilder) Build(c *fi.NodeupModelBuilderContext) error {
if err := b.AddCNIBinAssets(c); err != nil {
return err
if b.NodeupConfig.InstallCNIAssets {
if err := b.AddCNIBinAssets(c); err != nil {
return err
}
}

return nil
Expand Down
6 changes: 6 additions & 0 deletions pkg/apis/kops/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,12 @@ func (c *Cluster) UsesNoneDNS() bool {
return false
}

func (c *Cluster) InstallCNIAssets() bool {
return c.Spec.Networking.AmazonVPC == nil &&
c.Spec.Networking.Calico == nil &&
c.Spec.Networking.Cilium == nil
}

func (c *Cluster) APIInternalName() string {
return "api.internal." + c.ObjectMeta.Name
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/apis/kops/model/instance_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ type InstanceGroup interface {
// GetCloudProvider returns the cloud provider for the instance group
GetCloudProvider() kops.CloudProviderID

// InstallCNIAssets returns true if CNI network plugins need to be installed
InstallCNIAssets() bool

// RawClusterSpec returns the cluster spec for the instance group.
// If possible, prefer abstracted methods over accessing this data directly.
RawClusterSpec() *kops.ClusterSpec
Expand Down Expand Up @@ -67,6 +70,10 @@ func (m *instanceGroupModel) GetCloudProvider() kops.CloudProviderID {
return m.cluster.GetCloudProvider()
}

func (m *instanceGroupModel) InstallCNIAssets() bool {
return m.cluster.InstallCNIAssets()
}

func (m *instanceGroupModel) RawClusterSpec() *kops.ClusterSpec {
return &m.cluster.Spec
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/apis/nodeup/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ type Config struct {
KubeProxy *kops.KubeProxyConfig
// Networking configures networking.
Networking kops.NetworkingSpec
// InstallCNIAssets specifies that the CNI network plugins need to be installed.
InstallCNIAssets bool `json:",omitempty"`
// UseCiliumEtcd is true when a Cilium etcd cluster is present.
UseCiliumEtcd bool `json:",omitempty"`
// UsesKubenet specifies that the CNI is derived from Kubenet.
Expand Down Expand Up @@ -297,6 +299,10 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
config.UpdatePolicy = kops.UpdatePolicyAutomatic
}

if cluster.InstallCNIAssets() {
config.InstallCNIAssets = true
}

if cluster.Spec.Networking.AmazonVPC != nil {
config.Networking.AmazonVPC = &kops.AmazonVPCNetworkingSpec{}
config.DefaultMachineType = aws.String(strings.Split(instanceGroup.Spec.MachineType, ",")[0])
Expand Down
10 changes: 4 additions & 6 deletions pkg/assets/assetdata/cni.yaml → pkg/assets/assetdata/cni-0.9.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
# This file is generated by generate-asset-hashes.sh

filestores:
- base: https://storage.googleapis.com/k8s-artifacts-cni/release/

files:
# cni 0.9.1
- name: v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
sha256: 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7
# cni 0.9.1
- name: v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
sha256: ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0

- name: v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
sha256: f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37
- name: v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
sha256: 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57

12 changes: 12 additions & 0 deletions pkg/assets/assetdata/cni-1.2.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# This file is generated by generate-asset-hashes.sh

filestores:
- base: https://storage.googleapis.com/k8s-artifacts-cni/release/

files:
# cni 1.2.0
- name: v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
sha256: f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37
# cni 1.2.0
- name: v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
sha256: 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57
12 changes: 12 additions & 0 deletions pkg/assets/assetdata/cni-1.3.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# This file is generated by generate-asset-hashes.sh

filestores:
- base: https://storage.googleapis.com/k8s-artifacts-cni/release/

files:
# cni 1.3.0
- name: v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
sha256: 754a71ed60a4bd08726c3af705a7d55ee3df03122b12e389fdba4bea35d7dd7e
# cni 1.3.0
- name: v1.3.0/cni-plugins-linux-arm64-v1.3.0.tgz
sha256: de7a666fd6ad83a228086bd55756db62ef335a193d1b143d910b69f079e30598
12 changes: 12 additions & 0 deletions pkg/assets/assetdata/cni-1.4.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# This file is generated by generate-asset-hashes.sh

filestores:
- base: https://storage.googleapis.com/k8s-artifacts-cni/release/

files:
# cni 1.4.1
- name: v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz
sha256: 1511f6c003ace805eafeb1132727791326283cff88a923d76329e1892bba7a10
# cni 1.4.1
- name: v1.4.1/cni-plugins-linux-arm64-v1.4.1.tgz
sha256: 72644e13557cda8a5b39baf97fc5e93d23fdf7baba7700000e7e9efd8bdf9234
12 changes: 12 additions & 0 deletions pkg/assets/assetdata/cni-1.5.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# This file is generated by generate-asset-hashes.sh

filestores:
- base: https://storage.googleapis.com/k8s-artifacts-cni/release/

files:
# cni 1.5.1
- name: v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz
sha256: 77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b
# cni 1.5.1
- name: v1.5.1/cni-plugins-linux-arm64-v1.5.1.tgz
sha256: c2a292714d0fad98a3491ae43df8ad58354b3c0bdf5d5a3e281777967c70fcff
12 changes: 12 additions & 0 deletions pkg/assets/assetdata/cni-1.6.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# This file is generated by generate-asset-hashes.sh

filestores:
- base: https://storage.googleapis.com/k8s-artifacts-cni/release/

files:
# cni 1.6.1
- name: v1.6.1/cni-plugins-linux-amd64-v1.6.1.tgz
sha256: 2503ce29ac445715ebe146073f45468153f9e28f45fa173cb060cfd9e735f563
# cni 1.6.1
- name: v1.6.1/cni-plugins-linux-arm64-v1.6.1.tgz
sha256: f0f440b968ab50ad13d9d42d993ba98ec30b2ec666846f4ef1bddc7646a701cc
11 changes: 9 additions & 2 deletions pkg/assets/mirrors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ var wellKnownMirrors = []mirrorConfig{
"https://github.com/kubernetes/kops/releases/download/v{kopsVersion}/",
},
},
{
Base: "https://storage.googleapis.com/k8s-artifacts-cni/release/",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't we invert this?

long term we'd like to eliminate this host, we have been exiting making kubernetes responsible for hosting third party assets, especially on non-rate-limited hosts that we pay for (And it seems the main use case to not just fetch from github is bypassing their rate limits)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ref: kubernetes/k8s.io#7584

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, the main use case is bypassing the rate limits.
We started discussing inverting this today, but no conclusion yet. The initial intention was to keep this consistent and decide later which ones to change to the mirror as primary.
@justinsb any preference on how to approach this?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, the main use case is bypassing the rate limits.

Yeah, the problem is this then makes us responsible for no rate limit, I think if users are pulling so much that github is blocking them, then they probably need to use their own mirror or something?

We have been rate limiting other hosts like registry.k8s.io, but that's not possible here without moving to a different host (at which point ... just use github / official downloads?)

Mirrors: []string{
"https://github.com/containernetworking/plugins/releases/download/",
},
},
{
Base: "https://dl.k8s.io/release/",
Mirrors: []string{
Expand All @@ -65,10 +71,10 @@ func (m *mirrorConfig) findMirrors(u string) ([]string, bool) {
mirrors := []string{u}

for _, mirror := range m.Mirrors {
mirror = strings.ReplaceAll(mirror, "{kopsVersion}", kops.Version)
suffix := strings.TrimPrefix(u, baseURLString)

if strings.HasPrefix(mirror, "https://github.com") {
if strings.HasPrefix(mirror, "https://github.com") && strings.Contains(mirror, "/kops/") {
mirror = strings.ReplaceAll(mirror, "{kopsVersion}", kops.Version)
// GitHub artifact names are quite different, because the suffix path is collapsed.
suffix = strings.ReplaceAll(suffix, "/", "-")
suffix = strings.ReplaceAll(suffix, "linux-amd64-nodeup", "nodeup-linux-amd64")
Expand All @@ -78,6 +84,7 @@ func (m *mirrorConfig) findMirrors(u string) ([]string, bool) {
suffix = strings.ReplaceAll(suffix, "linux-amd64-channels", "channels-linux-amd64")
suffix = strings.ReplaceAll(suffix, "linux-arm64-channels", "channels-linux-arm64")
}

mirrors = append(mirrors, mirror+suffix)
}
return mirrors, true
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/tests/data/bootstrapscript_0.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: aws
InstanceGroupName: testIG
InstanceGroupRole: ControlPlane
NodeupConfigHash: sq0FwAxnWal2+vIUsu8xUKK8Q+Vzx3V9LKkSFo/ds4M=
NodeupConfigHash: fAxvqbU++fBpT6SIjMMeXWlQ4oHqL/M8N7AaaJkOPOM=

__EOF_KUBE_ENV

Expand Down
2 changes: 1 addition & 1 deletion pkg/model/tests/data/bootstrapscript_1.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: aws
InstanceGroupName: testIG
InstanceGroupRole: ControlPlane
NodeupConfigHash: soFi0PS8cYVnHNTGuj1Fv1d8Q71M6D9Mgo/fjlPSkB0=
NodeupConfigHash: N0d3Il7CeGYFi32ZDb4i3iftzUD6swJ3F/tZf8d1bZY=

__EOF_KUBE_ENV

Expand Down
2 changes: 1 addition & 1 deletion pkg/model/tests/data/bootstrapscript_2.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: aws
InstanceGroupName: testIG
InstanceGroupRole: ControlPlane
NodeupConfigHash: soFi0PS8cYVnHNTGuj1Fv1d8Q71M6D9Mgo/fjlPSkB0=
NodeupConfigHash: N0d3Il7CeGYFi32ZDb4i3iftzUD6swJ3F/tZf8d1bZY=

__EOF_KUBE_ENV

Expand Down
2 changes: 1 addition & 1 deletion pkg/model/tests/data/bootstrapscript_3.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: aws
InstanceGroupName: testIG
InstanceGroupRole: Node
NodeupConfigHash: bzOu75vonqETQp0my4RwAbbZvfTkGmwJ0uvaN5JHI5Y=
NodeupConfigHash: X6Di6WM7EcwYaitdtxqgDgb2VjQuX4gHgLLPBZUb6Hw=

__EOF_KUBE_ENV

Expand Down
2 changes: 1 addition & 1 deletion pkg/model/tests/data/bootstrapscript_4.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: aws
InstanceGroupName: testIG
InstanceGroupRole: Node
NodeupConfigHash: Pj5zYnkoZ3wGhph8FTN58SH0n4LL85thsUN6YE09xe0=
NodeupConfigHash: FOtsEbu1CrMgt9fSSoK3X+UvdHnVKS4MmNRxjGmd40c=

__EOF_KUBE_ENV

Expand Down
2 changes: 1 addition & 1 deletion pkg/model/tests/data/bootstrapscript_5.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: aws
InstanceGroupName: testIG
InstanceGroupRole: Node
NodeupConfigHash: Pj5zYnkoZ3wGhph8FTN58SH0n4LL85thsUN6YE09xe0=
NodeupConfigHash: FOtsEbu1CrMgt9fSSoK3X+UvdHnVKS4MmNRxjGmd40c=

__EOF_KUBE_ENV

Expand Down
1 change: 1 addition & 0 deletions pkg/model/tests/data/nodeupconfig_0.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ Hooks:
ExecStart=/usr/bin/systemctl start apply-to-all.service
name: apply-to-all.service
- null
InstallCNIAssets: true
KeypairIDs: {}
KubeProxy:
cpuLimit: 30m
Expand Down
1 change: 1 addition & 0 deletions pkg/model/tests/data/nodeupconfig_1.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ Hooks:
- -c
- apt-get update
image: busybox
InstallCNIAssets: true
KeypairIDs: {}
KubeProxy:
cpuLimit: 30m
Expand Down
1 change: 1 addition & 0 deletions pkg/model/tests/data/nodeupconfig_2.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ Hooks:
- -c
- apt-get update
image: busybox
InstallCNIAssets: true
KeypairIDs: {}
KubeProxy:
cpuLimit: 30m
Expand Down
1 change: 1 addition & 0 deletions pkg/model/tests/data/nodeupconfig_3.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Hooks:
ExecStart=/usr/bin/systemctl start apply-to-all.service
name: apply-to-all.service
- null
InstallCNIAssets: true
KeypairIDs: {}
KubeProxy:
cpuLimit: 30m
Expand Down
1 change: 1 addition & 0 deletions pkg/model/tests/data/nodeupconfig_4.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ Hooks:
- -c
- apt-get update
image: busybox
InstallCNIAssets: true
KeypairIDs: {}
KubeProxy:
cpuLimit: 30m
Expand Down
1 change: 1 addition & 0 deletions pkg/model/tests/data/nodeupconfig_5.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ Hooks:
- -c
- apt-get update
image: busybox
InstallCNIAssets: true
KeypairIDs: {}
KubeProxy:
cpuLimit: 30m
Expand Down
2 changes: 1 addition & 1 deletion pkg/nodemodel/fileassets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func BuildKubernetesFileAssets(ig model.InstanceGroup, assetBuilder *assets.Asse
}
}

{
if ig.InstallCNIAssets() {
cniAsset, err := wellknownassets.FindCNIAssets(ig, assetBuilder, arch)
if err != nil {
return nil, err
Expand Down
Loading
Loading