Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable using AWS spot instances #203

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Enable using AWS spot instances #203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions OCP-4.X/deploy-cluster.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
pre_tasks:
- name: Include platform variables
include_vars:
file: "vars/install-on-{{ {{ (platform == 'aws-arm') | ternary('aws', platform) }} }}.yml"
file: "vars/install-on-{{ (platform == 'aws-arm') | ternary('aws', platform) }}.yml"
roles:
- role: node-debug-config
when: openshift_debug_config|bool
Expand Down Expand Up @@ -125,7 +125,7 @@
pre_tasks:
- name: Include platform variables
include_vars:
file: "vars/install-on-{{ {{ (platform == 'aws-arm') | ternary('aws', platform) }} }}.yml"
file: "vars/install-on-{{ (platform == 'aws-arm') | ternary('aws', platform) }}.yml"
roles:
- role: node-debug-config
when: openshift_debug_config|bool
2 changes: 1 addition & 1 deletion OCP-4.X/destroy-cluster.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

- name: Include platform variables
include_vars:
file: "vars/install-on-{{ {{ (platform == 'aws-arm') | ternary('aws', platform) }} }}.yml"
file: vars/install-on-{{ (platform == 'aws-arm') | ternary('aws', platform) }}.yml
- name: Set dynamic scale-ci-deploy path
set_fact:
dynamic_deploy_path: "{% if lookup('env', 'DYNAMIC_DEPLOY_PATH') %}{{ lookup('env', 'DYNAMIC_DEPLOY_PATH') }}{% else %}scale-ci-{{ openshift_cluster_name }}-{{ platform }}{% endif %}"
Expand Down
10 changes: 10 additions & 0 deletions OCP-4.X/roles/openshift-install/tasks/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,16 @@

when: platform == "alibaba"

- name: Create openshift manifests
shell: |
set -o pipefail
cd {{ ansible_user_dir }}/{{ dynamic_deploy_path }}
export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE={{ openshift_install_release_image_override }}
export GOOGLE_CREDENTIALS={{ gcp_auth_key_file|default() }}
bin/openshift-install create manifests --dir .
sed -i -e '/ami:/i\ \spotMarketOptions: {}' {{ ansible_user_dir }}/{{ dynamic_deploy_path }}/openshift/99_openshift-cluster-api_worker*
when: platform == "aws" and spot_instance_zone

- name: Run openshift installer
shell: |
set -o pipefail
Expand Down
8 changes: 8 additions & 0 deletions OCP-4.X/roles/openshift-install/templates/install-config-aws.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ compute:
name: worker
platform:
aws:
{% if spot_instance_zone -%}
zones:
- {{spot_instance_zone}}
{%+ endif -%}
type: {{ openshift_worker_instance_type }}
rootVolume:
iops: {{ openshift_worker_root_volume_iops }}
Expand All @@ -16,6 +20,10 @@ ControlPlane:
name: master
platform:
aws:
{% if spot_instance_zone -%}
zones:
- {{spot_instance_zone}}
{%+ endif -%}
type: {{ openshift_master_instance_type }}
rootVolume:
iops: {{ openshift_master_root_volume_iops }}
Expand Down
12 changes: 10 additions & 2 deletions OCP-4.X/roles/post-install/tasks/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,13 @@
with_items:
- src: aws-infra-node-machineset.yml.j2
dest: "{{ ansible_user_dir }}/{{ dynamic_deploy_path }}/infra-node-machineset.yml"
toggle: "{{openshift_toggle_infra_node}}"
toggle: "{{openshift_toggle_infra_node and (spot_instance_zone==false or spot_instance_zone=='')}}"
- src: aws-workload-node-machineset.yml.j2
dest: "{{ ansible_user_dir }}/{{ dynamic_deploy_path }}/workload-node-machineset.yml"
toggle: "{{openshift_toggle_workload_node}}"
- src: aws-spot-infra-node-machineset.yml.j2
dest: "{{ ansible_user_dir }}/{{ dynamic_deploy_path }}/infra-node-machineset.yml"
toggle: "{{openshift_toggle_infra_node and spot_instance_zone!=false and spot_instance_zone!=''}}"
when: platform == "aws" or platform == "aws-arm"

- name: Azure Block of tasks
Expand Down Expand Up @@ -241,7 +244,12 @@
- name: Increment expected node count with infra nodes
set_fact:
expected_node_count: "{{expected_node_count|int + 3}}"
when: openshift_toggle_infra_node|bool
when: openshift_toggle_infra_node and (spot_instance_zone==false or spot_instance_zone=='')

- name: Increment expected node count with spot infra nodes
set_fact:
expected_node_count: "{{expected_node_count|int + 1}}"
when: openshift_toggle_infra_node and spot_instance_zone!=false and spot_instance_zone!=''

- name: Increment expected node count with workload node
set_fact:
Expand Down
76 changes: 76 additions & 0 deletions OCP-4.X/roles/post-install/templates/aws-spot-infra-node-machineset.yml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
apiVersion: v1
items:
- apiVersion: machine.openshift.io/v1beta1
kind: MachineSet
metadata:
creationTimestamp: null
labels:
{{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
{{machineset_metadata_label_prefix}}/cluster-api-machine-role: infra
{{machineset_metadata_label_prefix}}/cluster-api-machine-type: infra
name: infra-{{spot_instance_zone}}
namespace: openshift-machine-api
spec:
replicas: 1
selector:
matchLabels:
{{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
{{machineset_metadata_label_prefix}}/cluster-api-machineset: infra-{{spot_instance_zone}}
template:
metadata:
creationTimestamp: null
labels:
{{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
{{machineset_metadata_label_prefix}}/cluster-api-machine-role: infra
{{machineset_metadata_label_prefix}}/cluster-api-machine-type: infra
{{machineset_metadata_label_prefix}}/cluster-api-machineset: infra-{{spot_instance_zone}}
spec:
metadata:
creationTimestamp: null
labels:
node-role.kubernetes.io/infra: ""
providerSpec:
value:
spotMarketOptions: {}
ami:
id: {{ami_id.stdout}}
apiVersion: awsproviderconfig.openshift.io/v1beta1
blockDevices:
- ebs:
iops: {{openshift_infra_node_volume_iops}}
volumeSize: {{openshift_infra_node_volume_size}}
volumeType: {{openshift_infra_node_volume_type}}
credentialsSecret:
name: aws-cloud-credentials
deviceIndex: 0
iamInstanceProfile:
id: {{cluster_name.stdout}}-worker-profile
instanceType: {{openshift_infra_node_instance_type}}
kind: AWSMachineProviderConfig
metadata:
creationTimestamp: null
placement:
availabilityZone: {{spot_instance_zone}}
region: {{aws_region.stdout}}
publicIp: false
securityGroups:
- filters:
- name: tag:Name
values:
- {{cluster_name.stdout}}-worker-sg
subnet:
filters:
- name: tag:Name
values:
- {{cluster_name.stdout}}-private-{{spot_instance_zone}}
tags:
- name: kubernetes.io/cluster/{{cluster_name.stdout}}
value: owned
userDataSecret:
name: {{ user_data_secret }}
versions:
kubelet: ""
status:
replicas: 0
kind: List
metadata: {}
3 changes: 3 additions & 0 deletions OCP-4.X/roles/post-install/templates/aws-workload-node-machineset.yml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ items:
node-role.kubernetes.io/workload: ""
providerSpec:
value:
{% if spot_instance_zone -%}
spotMarketOptions: {}
{%+ endif -%}
ami:
id: {{ami_id.stdout}}
apiVersion: awsproviderconfig.openshift.io/v1beta1
Expand Down
3 changes: 2 additions & 1 deletion OCP-4.X/vars/install-on-aws.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
aws_region: "{{ lookup('env', 'AWS_REGION') }}"
aws_install_architecture: "{{ lookup('env', 'AWS_INSTALL_ARCHITECTURE') | default('amd64', true) }}"
cluster_owner: "{{ lookup('env', 'CLUSTER_OWNER') | default('ci', true) }}"
workload_aws_az_suffix: "{{ lookup('env', 'WORKLOAD_AWS_AZ_SUFFIX') | default('d', true) }}"
spot_instance_zone : "{{ lookup('env', 'SPOT_INSTANCE_ZONE')|default('', true) }}"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will limit availability zone to only what we specify, and in some cases even within a specific region 2 AZs have availability and one doesn't. Any thoughts on providing multiple values here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hello @smalleni While investigating the availability of a particular spot machine type in an AWS region, most of the time a specific spot machine type was available only in one zone in that region, so was able to test and hence base this patch on a single availability zone.
I'm considering adding the option to deploy across multiple availability zones once I get the chance to verify across multiple availability zones in a region.

workload_aws_az_suffix: "{{ spot_instance_zone[-1]|default(lookup('env', 'WORKLOAD_AWS_AZ_SUFFIX'),true)| default('d', true) }}"

# Cluster configuration
openshift_base_domain: "{{ lookup('env', 'OPENSHIFT_BASE_DOMAIN') }}"
Expand Down