[sai-gen] Support merging SAI specs. (sonic-net#577)

This change added the SAI spec merging support. This is to ensure any new SAI attribute changes will follow the rules below to help ABI compatibility:

- All new attributes or action enum values will be added in the end of the list.
- Existing attributes will be updated inline. In the future, we can add more check here, e.g., type changed in non-compatible way and etc.
- All old attributes will be marked as deprecated instead of removed.

Recently, the tunnel APIs are added, but because it merged with the SAI spec PR together without rebase, it is not captured in the previous SAI spec. With SAI spec merging, we can see the changes are showing up in the expected sequence. This would be an example that demos the several usages of SAI spec.

dash-pipeline/Makefile

@@ -177,8 +177,9 @@ sai: sai-headers sai-meta libsai
 sai-headers: p4 docker-saithrift-bldr-image-exists | SAI/SAI
 	@echo "Generate SAI library headers and implementation..."
 
-	# Once the specs are checked in, we can use this to revert any local changes before generating the new specs.
-	# git checkout SAI/specs/*
+	# Revert any local changes before generating the new specs.
+	git clean -xf SAI/specs
+	git checkout SAI/specs/*
 
 	mkdir -p SAI/lib
 

dash-pipeline/SAI/sai_api_gen.py

@@ -68,7 +68,8 @@
     print("Outputting new SAI spec to " + sai_spec_dir)
     yaml_inc_ctor.autoload = False
     new_sai_spec = dash_sai_exts.to_sai()
-    new_sai_spec.serialize(sai_spec_dir)
+    sai_spec.merge(new_sai_spec)
+    sai_spec.serialize(sai_spec_dir)
 
     # Generate and update all SAI files
     SAIGenerator(dash_sai_exts).generate()

dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml

@@ -217,4 +217,16 @@ sai_apis:
     allow_null: false
     valid_only: null
     deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_OUTBOUND_CA_TO_PA_ENTRY_DASH_TUNNEL_ID
+    description: Action parameter DASH_TUNNEL_ID
+    type: sai_object_id_t
+    attr_value_field: u16
+    default: SAI_NULL_OBJECT_ID
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: SAI_OBJECT_TYPE_DASH_TUNNEL
+    allow_null: true
+    valid_only: null
+    deprecated: null
   stats: []

dash-pipeline/SAI/specs/dash_outbound_routing.yaml

@@ -241,4 +241,16 @@ sai_apis:
     allow_null: false
     valid_only: null
     deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_OUTBOUND_ROUTING_ENTRY_DASH_TUNNEL_ID
+    description: Action parameter DASH_TUNNEL_ID
+    type: sai_object_id_t
+    attr_value_field: u16
+    default: SAI_NULL_OBJECT_ID
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: SAI_OBJECT_TYPE_DASH_TUNNEL
+    allow_null: true
+    valid_only: null
+    deprecated: null
   stats: []

dash-pipeline/SAI/specs/dash_tunnel.yaml

@@ -0,0 +1,48 @@
+!!python/object:utils.sai_spec.sai_api_group.SaiApiGroup
+name: dash_tunnel
+description: ''
+sai_apis:
+- !!python/object:utils.sai_spec.sai_api.SaiApi
+  name: dash_tunnel
+  description: ''
+  is_object: true
+  enums: []
+  structs: []
+  attributes:
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_TUNNEL_DIP
+    description: Action parameter DIP
+    type: sai_ip_address_t
+    attr_value_field: ipaddr
+    default: 0.0.0.0
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_TUNNEL_DASH_ENCAPSULATION
+    description: Action parameter DASH_ENCAPSULATION
+    type: sai_dash_encapsulation_t
+    attr_value_field: s32
+    default: SAI_DASH_ENCAPSULATION_VXLAN
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_TUNNEL_TUNNEL_KEY
+    description: Action parameter TUNNEL_KEY
+    type: sai_uint32_t
+    attr_value_field: u32
+    default: '0'
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  stats: []

dash-pipeline/SAI/specs/sai_spec.yaml

@@ -12,6 +12,7 @@ api_types:
 - SAI_API_DASH_PA_VALIDATION
 - SAI_API_ROUTE
 - SAI_API_DASH_VIP
+- SAI_API_DASH_TUNNEL
 object_types:
 - SAI_OBJECT_TYPE_DASH_ACL_GROUP
 - SAI_OBJECT_TYPE_DASH_ACL_RULE
@@ -30,6 +31,7 @@ object_types:
 - SAI_OBJECT_TYPE_PA_VALIDATION_ENTRY
 - SAI_OBJECT_TYPE_ROUTE_ENTRY
 - SAI_OBJECT_TYPE_VIP_ENTRY
+- SAI_OBJECT_TYPE_DASH_TUNNEL
 object_entries:
 - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
   name: direction_lookup_entry
@@ -97,81 +99,81 @@ enums:
     description: ''
     value: '2'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_tunnel_dscp_mode_t
+  name: sai_dash_encapsulation_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: PRESERVE_MODEL
+    name: INVALID
     description: ''
     value: '0'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: PIPE_MODEL
+    name: VXLAN
     description: ''
     value: '1'
+  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+    name: NVGRE
+    description: ''
+    value: '2'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_ha_role_t
+  name: sai_dash_routing_actions_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: DEAD
-    description: ''
-    value: '0'
-  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: ACTIVE
+    name: STATIC_ENCAP
     description: ''
     value: '1'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: STANDBY
+    name: NAT
     description: ''
     value: '2'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: STANDALONE
+    name: NAT46
     description: ''
-    value: '3'
+    value: '4'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: SWITCHING_TO_ACTIVE
+    name: NAT64
     description: ''
-    value: '4'
+    value: '8'
+  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+    name: NAT_PORT
+    description: ''
+    value: '16'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_encapsulation_t
+  name: sai_dash_tunnel_dscp_mode_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: INVALID
+    name: PRESERVE_MODEL
     description: ''
     value: '0'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: VXLAN
+    name: PIPE_MODEL
     description: ''
     value: '1'
-  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NVGRE
-    description: ''
-    value: '2'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_routing_actions_t
+  name: sai_dash_ha_role_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: STATIC_ENCAP
+    name: DEAD
     description: ''
-    value: '1'
+    value: '0'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT
+    name: ACTIVE
     description: ''
-    value: '2'
+    value: '1'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT46
+    name: STANDBY
     description: ''
-    value: '4'
+    value: '2'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT64
+    name: STANDALONE
     description: ''
-    value: '8'
+    value: '3'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT_PORT
+    name: SWITCHING_TO_ACTIVE
     description: ''
-    value: '16'
+    value: '4'
 port_extenstion: !!python/object:utils.sai_spec.sai_api_extension.SaiApiExtension
   attributes: []
   stats:
@@ -260,3 +262,4 @@ api_groups:
 - !inc '/SAI/specs/dash_pa_validation.yaml'
 - !inc '/SAI/specs/route.yaml'
 - !inc '/SAI/specs/dash_vip.yaml'
+- !inc '/SAI/specs/dash_tunnel.yaml'

dash-pipeline/SAI/utils/sai_spec/sai_api.py

@@ -3,6 +3,7 @@
 from .sai_attribute import SaiAttribute
 from .sai_enum import SaiEnum
 from .sai_struct import SaiStruct
+from . import sai_spec_utils
 
 
 class SaiApi(SaiCommon):
@@ -17,3 +18,12 @@ def __init__(self, name: str, description: str, is_object: bool = False):
         self.structs: List[SaiStruct] = []
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+
+        self.is_object = other.is_object
+        sai_spec_utils.merge_sai_common_lists(self.enums, other.enums)
+        sai_spec_utils.merge_sai_common_lists(self.structs, other.structs)
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)

dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py

@@ -1,5 +1,6 @@
 from typing import List
 from .sai_attribute import SaiAttribute
+from . import sai_spec_utils
 
 
 class SaiApiExtension:
@@ -12,3 +13,7 @@ class SaiApiExtension:
     def __init__(self):
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
+
+    def merge(self, other: "SaiApiExtension"):
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)

dash-pipeline/SAI/utils/sai_spec/sai_api_group.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_api import SaiApi
+from . import sai_spec_utils
 
 
 class SaiApiGroup(SaiCommon):
@@ -11,3 +12,20 @@ class SaiApiGroup(SaiCommon):
     def __init__(self, name: str, description: str):
         super().__init__(name, description)
         self.sai_apis: List[SaiApi] = []
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.sai_apis, other.sai_apis)
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate API group.
+
+        When deprecating the API group, we can safely remove it from the list as the
+        net effect is the same as keeping it:
+        - The old API type, object type and object entries will not be changed.
+        - The SAI headers will not be changed, because their API groups are present.
+        - The DASH libsai will not be generated anymore, but it is ok, since we will not
+          use them in the BMv2 anyway.
+        """
+        return True

dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py

@@ -0,0 +1,19 @@
+from typing import Dict, List
+
+
+class SaiApiP4MetaAction:
+    def __init__(self, name: str, id: int):
+        self.name: str = name
+        self.id: int = id
+        self.attr_param_id: Dict[str, int] = {}
+
+
+class SaiApiP4MetaTable:
+    def __init__(self, id: int):
+        self.id: int = id
+        self.actions: Dict[str, SaiApiP4MetaAction] = {}
+
+
+class SaiApiP4Meta:
+    def __init__(self):
+        self.tables: List[SaiApiP4MetaTable] = []

dash-pipeline/SAI/utils/sai_spec/sai_attribute.py

@@ -31,3 +31,11 @@ def __init__(
         self.allow_null = allow_null
         self.valid_only = valid_only
         self.deprecated = deprecated
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.__dict__.update(other.__dict__)
+
+    def deprecate(self) -> bool:
+        self.deprecated = True
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_common.py

@@ -6,3 +6,21 @@ class SaiCommon:
     def __init__(self, name: str, description: str):
         self.name: str = name
         self.description: str = description
+
+    def merge(self, other: "SaiCommon"):
+        """
+        Merge the other SaiCommon object into this object.
+        """
+        if not isinstance(other, type(self)):
+            raise TypeError(f"Cannot merge {type(self)} with {type(other)}")
+
+        self.description = other.description
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate this object.
+        
+        If the value doesn't support deprecation marking, we don't do anything
+        but return False to keep it in the list.
+        """
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_enum.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_enum_member import SaiEnumMember
+from . import sai_spec_utils
 
 
 class SaiEnum(SaiCommon):
@@ -11,3 +12,7 @@ class SaiEnum(SaiCommon):
     def __init__(self, name: str, description: str, members: List[SaiEnumMember] = []):
         super().__init__(name, description)
         self.members: List[SaiEnumMember] = members
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.members, other.members)

dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py

@@ -10,3 +10,7 @@ class SaiEnumMember(SaiCommon):
     def __init__(self, name: str, description: str, value: str):
         super().__init__(name, description)
         self.value: str = value
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.value = other.value