diff --git a/dash-pipeline/Makefile b/dash-pipeline/Makefile index 89622a1dd..fe7c42680 100644 --- a/dash-pipeline/Makefile +++ b/dash-pipeline/Makefile @@ -177,8 +177,9 @@ sai: sai-headers sai-meta libsai sai-headers: p4 docker-saithrift-bldr-image-exists | SAI/SAI @echo "Generate SAI library headers and implementation..." - # Once the specs are checked in, we can use this to revert any local changes before generating the new specs. - # git checkout SAI/specs/* + # Revert any local changes before generating the new specs. + git clean -xf SAI/specs + git checkout SAI/specs/* mkdir -p SAI/lib diff --git a/dash-pipeline/SAI/sai_api_gen.py b/dash-pipeline/SAI/sai_api_gen.py index af8098b48..ceb574d42 100755 --- a/dash-pipeline/SAI/sai_api_gen.py +++ b/dash-pipeline/SAI/sai_api_gen.py @@ -68,7 +68,8 @@ print("Outputting new SAI spec to " + sai_spec_dir) yaml_inc_ctor.autoload = False new_sai_spec = dash_sai_exts.to_sai() - new_sai_spec.serialize(sai_spec_dir) + sai_spec.merge(new_sai_spec) + sai_spec.serialize(sai_spec_dir) # Generate and update all SAI files SAIGenerator(dash_sai_exts).generate() diff --git a/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml b/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml index ffdf34798..41a7489bf 100644 --- a/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml +++ b/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml @@ -217,4 +217,16 @@ sai_apis: allow_null: false valid_only: null deprecated: null + - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute + name: SAI_OUTBOUND_CA_TO_PA_ENTRY_DASH_TUNNEL_ID + description: Action parameter DASH_TUNNEL_ID + type: sai_object_id_t + attr_value_field: u16 + default: SAI_NULL_OBJECT_ID + isresourcetype: false + flags: CREATE_AND_SET + object_name: SAI_OBJECT_TYPE_DASH_TUNNEL + allow_null: true + valid_only: null + deprecated: null stats: [] diff --git a/dash-pipeline/SAI/specs/dash_outbound_routing.yaml b/dash-pipeline/SAI/specs/dash_outbound_routing.yaml index c97f30b36..10ff530f2 100644 --- a/dash-pipeline/SAI/specs/dash_outbound_routing.yaml +++ b/dash-pipeline/SAI/specs/dash_outbound_routing.yaml @@ -241,4 +241,16 @@ sai_apis: allow_null: false valid_only: null deprecated: null + - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute + name: SAI_OUTBOUND_ROUTING_ENTRY_DASH_TUNNEL_ID + description: Action parameter DASH_TUNNEL_ID + type: sai_object_id_t + attr_value_field: u16 + default: SAI_NULL_OBJECT_ID + isresourcetype: false + flags: CREATE_AND_SET + object_name: SAI_OBJECT_TYPE_DASH_TUNNEL + allow_null: true + valid_only: null + deprecated: null stats: [] diff --git a/dash-pipeline/SAI/specs/dash_tunnel.yaml b/dash-pipeline/SAI/specs/dash_tunnel.yaml new file mode 100644 index 000000000..524ff8210 --- /dev/null +++ b/dash-pipeline/SAI/specs/dash_tunnel.yaml @@ -0,0 +1,48 @@ +!!python/object:utils.sai_spec.sai_api_group.SaiApiGroup +name: dash_tunnel +description: '' +sai_apis: +- !!python/object:utils.sai_spec.sai_api.SaiApi + name: dash_tunnel + description: '' + is_object: true + enums: [] + structs: [] + attributes: + - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute + name: SAI_DASH_TUNNEL_DIP + description: Action parameter DIP + type: sai_ip_address_t + attr_value_field: ipaddr + default: 0.0.0.0 + isresourcetype: false + flags: CREATE_AND_SET + object_name: null + allow_null: false + valid_only: null + deprecated: null + - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute + name: SAI_DASH_TUNNEL_DASH_ENCAPSULATION + description: Action parameter DASH_ENCAPSULATION + type: sai_dash_encapsulation_t + attr_value_field: s32 + default: SAI_DASH_ENCAPSULATION_VXLAN + isresourcetype: false + flags: CREATE_AND_SET + object_name: null + allow_null: false + valid_only: null + deprecated: null + - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute + name: SAI_DASH_TUNNEL_TUNNEL_KEY + description: Action parameter TUNNEL_KEY + type: sai_uint32_t + attr_value_field: u32 + default: '0' + isresourcetype: false + flags: CREATE_AND_SET + object_name: null + allow_null: false + valid_only: null + deprecated: null + stats: [] diff --git a/dash-pipeline/SAI/specs/sai_spec.yaml b/dash-pipeline/SAI/specs/sai_spec.yaml index 47b34d09b..76e5b6129 100644 --- a/dash-pipeline/SAI/specs/sai_spec.yaml +++ b/dash-pipeline/SAI/specs/sai_spec.yaml @@ -12,6 +12,7 @@ api_types: - SAI_API_DASH_PA_VALIDATION - SAI_API_ROUTE - SAI_API_DASH_VIP +- SAI_API_DASH_TUNNEL object_types: - SAI_OBJECT_TYPE_DASH_ACL_GROUP - SAI_OBJECT_TYPE_DASH_ACL_RULE @@ -30,6 +31,7 @@ object_types: - SAI_OBJECT_TYPE_PA_VALIDATION_ENTRY - SAI_OBJECT_TYPE_ROUTE_ENTRY - SAI_OBJECT_TYPE_VIP_ENTRY +- SAI_OBJECT_TYPE_DASH_TUNNEL object_entries: - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry name: direction_lookup_entry @@ -97,81 +99,81 @@ enums: description: '' value: '2' - !!python/object:utils.sai_spec.sai_enum.SaiEnum - name: sai_dash_tunnel_dscp_mode_t + name: sai_dash_encapsulation_t description: '' members: - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: PRESERVE_MODEL + name: INVALID description: '' value: '0' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: PIPE_MODEL + name: VXLAN description: '' value: '1' + - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember + name: NVGRE + description: '' + value: '2' - !!python/object:utils.sai_spec.sai_enum.SaiEnum - name: sai_dash_ha_role_t + name: sai_dash_routing_actions_t description: '' members: - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: DEAD - description: '' - value: '0' - - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: ACTIVE + name: STATIC_ENCAP description: '' value: '1' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: STANDBY + name: NAT description: '' value: '2' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: STANDALONE + name: NAT46 description: '' - value: '3' + value: '4' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: SWITCHING_TO_ACTIVE + name: NAT64 description: '' - value: '4' + value: '8' + - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember + name: NAT_PORT + description: '' + value: '16' - !!python/object:utils.sai_spec.sai_enum.SaiEnum - name: sai_dash_encapsulation_t + name: sai_dash_tunnel_dscp_mode_t description: '' members: - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: INVALID + name: PRESERVE_MODEL description: '' value: '0' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: VXLAN + name: PIPE_MODEL description: '' value: '1' - - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: NVGRE - description: '' - value: '2' - !!python/object:utils.sai_spec.sai_enum.SaiEnum - name: sai_dash_routing_actions_t + name: sai_dash_ha_role_t description: '' members: - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: STATIC_ENCAP + name: DEAD description: '' - value: '1' + value: '0' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: NAT + name: ACTIVE description: '' - value: '2' + value: '1' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: NAT46 + name: STANDBY description: '' - value: '4' + value: '2' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: NAT64 + name: STANDALONE description: '' - value: '8' + value: '3' - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember - name: NAT_PORT + name: SWITCHING_TO_ACTIVE description: '' - value: '16' + value: '4' port_extenstion: !!python/object:utils.sai_spec.sai_api_extension.SaiApiExtension attributes: [] stats: @@ -260,3 +262,4 @@ api_groups: - !inc '/SAI/specs/dash_pa_validation.yaml' - !inc '/SAI/specs/route.yaml' - !inc '/SAI/specs/dash_vip.yaml' +- !inc '/SAI/specs/dash_tunnel.yaml' diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api.py b/dash-pipeline/SAI/utils/sai_spec/sai_api.py index 07334c4f9..494c11bc5 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_api.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_api.py @@ -3,6 +3,7 @@ from .sai_attribute import SaiAttribute from .sai_enum import SaiEnum from .sai_struct import SaiStruct +from . import sai_spec_utils class SaiApi(SaiCommon): @@ -17,3 +18,12 @@ def __init__(self, name: str, description: str, is_object: bool = False): self.structs: List[SaiStruct] = [] self.attributes: List[SaiAttribute] = [] self.stats: List[SaiAttribute] = [] + + def merge(self, other: "SaiCommon"): + super().merge(other) + + self.is_object = other.is_object + sai_spec_utils.merge_sai_common_lists(self.enums, other.enums) + sai_spec_utils.merge_sai_common_lists(self.structs, other.structs) + sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes) + sai_spec_utils.merge_sai_common_lists(self.stats, other.stats) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py index 3ebd9e38a..d3324d88c 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py @@ -1,5 +1,6 @@ from typing import List from .sai_attribute import SaiAttribute +from . import sai_spec_utils class SaiApiExtension: @@ -12,3 +13,7 @@ class SaiApiExtension: def __init__(self): self.attributes: List[SaiAttribute] = [] self.stats: List[SaiAttribute] = [] + + def merge(self, other: "SaiApiExtension"): + sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes) + sai_spec_utils.merge_sai_common_lists(self.stats, other.stats) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py index 25f0b7b38..b960a9f99 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py @@ -1,6 +1,7 @@ from typing import List from .sai_common import SaiCommon from .sai_api import SaiApi +from . import sai_spec_utils class SaiApiGroup(SaiCommon): @@ -11,3 +12,20 @@ class SaiApiGroup(SaiCommon): def __init__(self, name: str, description: str): super().__init__(name, description) self.sai_apis: List[SaiApi] = [] + + def merge(self, other: "SaiCommon"): + super().merge(other) + sai_spec_utils.merge_sai_common_lists(self.sai_apis, other.sai_apis) + + def deprecate(self) -> bool: + """ + Deprecate API group. + + When deprecating the API group, we can safely remove it from the list as the + net effect is the same as keeping it: + - The old API type, object type and object entries will not be changed. + - The SAI headers will not be changed, because their API groups are present. + - The DASH libsai will not be generated anymore, but it is ok, since we will not + use them in the BMv2 anyway. + """ + return True diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py new file mode 100644 index 000000000..e1279faeb --- /dev/null +++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py @@ -0,0 +1,19 @@ +from typing import Dict, List + + +class SaiApiP4MetaAction: + def __init__(self, name: str, id: int): + self.name: str = name + self.id: int = id + self.attr_param_id: Dict[str, int] = {} + + +class SaiApiP4MetaTable: + def __init__(self, id: int): + self.id: int = id + self.actions: Dict[str, SaiApiP4MetaAction] = {} + + +class SaiApiP4Meta: + def __init__(self): + self.tables: List[SaiApiP4MetaTable] = [] diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py b/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py index bdb680f0a..88b0033ff 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py @@ -31,3 +31,11 @@ def __init__( self.allow_null = allow_null self.valid_only = valid_only self.deprecated = deprecated + + def merge(self, other: "SaiCommon"): + super().merge(other) + self.__dict__.update(other.__dict__) + + def deprecate(self) -> bool: + self.deprecated = True + return False \ No newline at end of file diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_common.py b/dash-pipeline/SAI/utils/sai_spec/sai_common.py index 7f1a21332..c6ee8e8cd 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_common.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_common.py @@ -6,3 +6,21 @@ class SaiCommon: def __init__(self, name: str, description: str): self.name: str = name self.description: str = description + + def merge(self, other: "SaiCommon"): + """ + Merge the other SaiCommon object into this object. + """ + if not isinstance(other, type(self)): + raise TypeError(f"Cannot merge {type(self)} with {type(other)}") + + self.description = other.description + + def deprecate(self) -> bool: + """ + Deprecate this object. + + If the value doesn't support deprecation marking, we don't do anything + but return False to keep it in the list. + """ + return False diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_enum.py b/dash-pipeline/SAI/utils/sai_spec/sai_enum.py index bae1cff4c..621b80427 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_enum.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_enum.py @@ -1,6 +1,7 @@ from typing import List from .sai_common import SaiCommon from .sai_enum_member import SaiEnumMember +from . import sai_spec_utils class SaiEnum(SaiCommon): @@ -11,3 +12,7 @@ class SaiEnum(SaiCommon): def __init__(self, name: str, description: str, members: List[SaiEnumMember] = []): super().__init__(name, description) self.members: List[SaiEnumMember] = members + + def merge(self, other: "SaiCommon"): + super().merge(other) + sai_spec_utils.merge_sai_common_lists(self.members, other.members) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py b/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py index f8cfb6bd2..fe4a458c8 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py @@ -10,3 +10,7 @@ class SaiEnumMember(SaiCommon): def __init__(self, name: str, description: str, value: str): super().__init__(name, description) self.value: str = value + + def merge(self, other: "SaiCommon"): + super().merge(other) + self.value = other.value \ No newline at end of file diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_spec.py b/dash-pipeline/SAI/utils/sai_spec/sai_spec.py index 74cd6ee33..e82faee03 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_spec.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_spec.py @@ -6,6 +6,7 @@ from .sai_api_group import SaiApiGroup from .sai_api_extension import SaiApiExtension from .sai_struct_entry import SaiStructEntry +from . import sai_spec_utils class SaiSpec: @@ -24,12 +25,16 @@ def __init__(self): def serialize(self, spec_dir: str): yaml_inc_files = [] for api_group in self.api_groups: - sai_api_group_spec_file_path = os.path.join(spec_dir, api_group.name + ".yaml") + sai_api_group_spec_file_path = os.path.join( + spec_dir, api_group.name + ".yaml" + ) with open(sai_api_group_spec_file_path, "w") as f: f.write(yaml.dump(api_group, indent=2, sort_keys=False)) - - yaml_inc_files.append(yaml_include.Data(urlpath=sai_api_group_spec_file_path)) + + yaml_inc_files.append( + yaml_include.Data(urlpath=sai_api_group_spec_file_path) + ) api_groups = self.api_groups self.api_groups = yaml_inc_files @@ -43,3 +48,21 @@ def serialize(self, spec_dir: str): def deserialize(spec_dir: str): with open(os.path.join(spec_dir, "sai_spec.yaml")) as f: return yaml.unsafe_load(f) + + def merge(self, other: "SaiSpec"): + sai_spec_utils.merge_sai_value_lists( + self.api_types, other.api_types, lambda x: x + ) + sai_spec_utils.merge_sai_value_lists( + self.object_types, other.object_types, lambda x: x + ) + sai_spec_utils.merge_sai_common_lists(self.object_entries, other.object_entries) + + # The global enums are generated from the P4 enum types, so we can respect whatever in the + # new spec and simply replace them, because: + # - It doesn't matter if the order of enum itself changes. + # - We cannot move the enum members as we want, as their order changes their values. + self.enums = other.enums + + self.port_extenstion.merge(other.port_extenstion) + sai_spec_utils.merge_sai_common_lists(self.api_groups, other.api_groups) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py b/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py new file mode 100644 index 000000000..747d19933 --- /dev/null +++ b/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py @@ -0,0 +1,50 @@ +from typing import Any, List, Callable +from .sai_common import SaiCommon + + +def merge_sai_value_lists( + target: List[Any], + source: List[Any], + get_key: Callable[[Any], str], + on_conflict: Callable[[Any, Any], None] = lambda x, y: x, + on_deprecate: Callable[[Any], bool] = lambda x: False +) -> None: + """ + Merge 2 SAI value lists from source list into target. + + Since we could not remove the old value or change the order of old values, the merge + is done as below: + - Any new values will be added in the end of the list. + - Any values that collapse with existing values will invoke on_conflict callback to resolve. + - Any values that needs to be removed will invoke on_deprecate function to deprecate. By default, + it will not be removed from the old list. + """ + target_dict = {get_key(item): item for item in target} + + source_keys = set() + for source_item in source: + source_key = get_key(source_item) + source_keys.add(source_key) + + if source_key in target_dict: + target_item = target_dict[source_key] + on_conflict(target_item, source_item) + else: + target.append(source_item) + target_dict[source_key] = source_item + + # Remove all items in target, if its key doesn't exist in source_keys and on_deprecate returns True. + target[:] = [item for item in target if get_key(item) in source_keys or not on_deprecate(item)] + + +def merge_sai_common_lists( + target: List[SaiCommon], + source: List[SaiCommon], +) -> None: + merge_sai_value_lists( + target, + source, + get_key=lambda x: x.name, + on_conflict=lambda x, y: x.merge(y), + on_deprecate=lambda x: x.deprecate(), + ) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_struct.py b/dash-pipeline/SAI/utils/sai_spec/sai_struct.py index b98cf3a27..025d1ecb4 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_struct.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_struct.py @@ -1,7 +1,7 @@ from typing import List from .sai_common import SaiCommon from .sai_struct_entry import SaiStructEntry - +from . import sai_spec_utils class SaiStruct(SaiCommon): """ @@ -11,3 +11,7 @@ class SaiStruct(SaiCommon): def __init__(self, name: str, description: str, members: List[SaiStructEntry] = []): super().__init__(name, description) self.members: List[SaiStructEntry] = members + + def merge(self, other: "SaiCommon"): + super().merge(other) + sai_spec_utils.merge_sai_common_lists(self.members, other.members) \ No newline at end of file diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py b/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py index 1f4e6ea06..ba3e42f99 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py @@ -19,3 +19,7 @@ def __init__( self.type = type self.objects = objects self.valid_only = valid_only + + def merge(self, other: "SaiCommon"): + super().merge(other) + self.__dict__.update(other.__dict__) \ No newline at end of file