diff --git a/Esm/ib/include/fm_xml.h b/Esm/ib/include/fm_xml.h index e46ea2b2..c45749b0 100644 --- a/Esm/ib/include/fm_xml.h +++ b/Esm/ib/include/fm_xml.h @@ -749,6 +749,36 @@ typedef struct _SmDGRouting_t { XMLMember_t dg[MAX_DGROUTING_ORDER]; } SmDGRouting_t; +/* + * Structures for (DG)SP routing control, per switch. + */ +typedef uint32_t portMap_t; + +static __inline__ void portMapSet(portMap_t *portMap, int port) +{ + portMap[port/(sizeof(portMap_t)*8)] |= (1u << port%(sizeof(portMap_t)*8)); +} + +static __inline__ portMap_t portMapTest(portMap_t *portMap, int port) +{ + return portMap[port/(sizeof(portMap_t)*8)] & (1u << port%(sizeof(portMap_t)*8)); +} + +typedef struct _SmSPRoutingPort { + uint8_t pport; + uint8_t vport; + uint16_t cost; +} SmSPRoutingPort_t; + +typedef struct _SmSPRoutingCtrl { + struct _SmSPRoutingCtrl *next; + uint64_t guid; + portMap_t pportMap[STL_MAX_PORTS/(sizeof(portMap_t)*8) + 1]; + portMap_t vportMap[STL_MAX_PORTS/(sizeof(portMap_t)*8) + 1]; + SmSPRoutingPort_t *ports; + uint16_t portCount; +} SmSPRoutingCtrl_t; + #ifdef CONFIG_INCLUDE_DOR #define MAX_TOROIDAL_DIMENSIONS 3 #define MAX_DOR_DIMENSIONS 20 @@ -852,6 +882,7 @@ typedef struct _SMXmlConfig { uint32_t node_appearance_msg_thresh; uint32_t spine_first_routing; uint32_t shortestPathBalanced; + uint32_t hypercube; uint32_t lmc; uint32_t lmc_e0; char routing_algorithm[STRING_SIZE]; @@ -943,6 +974,7 @@ typedef struct _SMXmlConfig { SmFtreeRouting_t ftreeRouting; SmDGRouting_t dgRouting; + SmSPRoutingCtrl_t *SPRoutingCtrl; char name[MAX_VFABRIC_NAME + 1]; uint32_t start; diff --git a/Esm/ib/src/config/fm_xml.c b/Esm/ib/src/config/fm_xml.c index 26b4e4f7..60da216e 100644 --- a/Esm/ib/src/config/fm_xml.c +++ b/Esm/ib/src/config/fm_xml.c @@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "if3.h" +#include #ifndef __VXWORKS__ @@ -100,6 +101,8 @@ uint32_t scpInProcess = 0; #define add_quotes(x) stringize(x) #endif +#define PORTS_ALLOC_UNIT 8 + #ifndef __VXWORKS__ int af_licensed = 0; extern int kmAdvancedFeatureVerification(char* keyString); @@ -1466,15 +1469,40 @@ void feShowConfig(FEXmlConfig_t *fep) printf("XML - subnet_size %u\n", (unsigned int)fep->subnet_size); } +void smFreeSPRoutingCtrl(SMXmlConfig_t *smp) +{ + SmSPRoutingCtrl_t *SPRoutingCtrl; + + SPRoutingCtrl = smp->SPRoutingCtrl; + if (SPRoutingCtrl == (void *)~0ul) { + SPRoutingCtrl = NULL; + } + while (SPRoutingCtrl) { + int portsSize = ROUNDUP(SPRoutingCtrl->portCount, PORTS_ALLOC_UNIT); + + if (portsSize) { + freeXmlMemory(SPRoutingCtrl->ports, portsSize*sizeof(*SPRoutingCtrl->ports), + "SmSPRoutingPort_t *ports"); + } + freeXmlMemory(SPRoutingCtrl, sizeof(SmSPRoutingCtrl_t), "SmSPRoutingCtrl_t *SPRoutingCtrl"); + SPRoutingCtrl = SPRoutingCtrl->next; + } + smp->SPRoutingCtrl = NULL; +} + // Free SM configs void smFreeConfig(SMXmlConfig_t *smp) { + smFreeSPRoutingCtrl(smp); freeXmlMemory(smp, sizeof(SMXmlConfig_t), "SMXmlConfig_t smFreeConfig()"); } // Copy SM configs boolean smCopyConfig(SMXmlConfig_t *dst, SMXmlConfig_t *src) { + SmSPRoutingCtrl_t *SPRoutingCtrl; + SmSPRoutingCtrl_t **prev; + if (!src || !dst) return 0; @@ -1483,6 +1511,42 @@ boolean smCopyConfig(SMXmlConfig_t *dst, SMXmlConfig_t *src) // Copy everything first, then handle the qmap *dst = *src; + + // Make a copy of the SPRoutingCtrl + SPRoutingCtrl = src->SPRoutingCtrl; + prev = &dst->SPRoutingCtrl; + if (SPRoutingCtrl == (void *)~0ul) { + SPRoutingCtrl = NULL; + } + while (SPRoutingCtrl) { + int portsSize = ROUNDUP(SPRoutingCtrl->portCount, PORTS_ALLOC_UNIT); + SmSPRoutingCtrl_t *dstSPRoutingCtrl; + + dstSPRoutingCtrl = getXmlMemory(sizeof(*dstSPRoutingCtrl), "SmSPRoutingCtrl_t *dstSPRoutingCtrl"); + if (!dstSPRoutingCtrl) { + fprintf(stderr, "%s: Memory limit exceeded for dstSPRoutingCtrl\n", + __FUNCTION__); + return FALSE; + } + memcpy(dstSPRoutingCtrl, SPRoutingCtrl, sizeof(*dstSPRoutingCtrl)); + if (portsSize) { + SmSPRoutingPort_t *ports; + + ports = getXmlMemory(portsSize*sizeof(*ports), "SmSPRoutingPort_t *ports"); + if (!ports) { + fprintf(stderr, "%s: Memory limit exceeded for ports\n", + __FUNCTION__); + return FALSE; + } + memcpy(ports, SPRoutingCtrl->ports, portsSize*sizeof(*ports)); + dstSPRoutingCtrl->ports = ports; + } + *prev = dstSPRoutingCtrl; + prev = &dstSPRoutingCtrl->next; + + SPRoutingCtrl = SPRoutingCtrl->next; + } + return 1; } @@ -1511,6 +1575,8 @@ void smClearConfig(SMXmlConfig_t *smp) memset(&smp->dgRouting.dg[i], 0, sizeof(smp->dgRouting.dg[i])); } + smFreeSPRoutingCtrl(smp); + // assume Sm is starting unless overridden or FM parent is not starting smp->start = 1; } @@ -1520,6 +1586,7 @@ void smInitConfig(SMXmlConfig_t *smp, SMDPLXmlConfig_t *dplp, SMMcastConfig_t *m { DGConfig_t *dg; uint32_t i; + SmSPRoutingCtrl_t *SPRoutingCtrl; if (!smp || !dplp || !mcp || !mlsp || !dgp) return; @@ -1559,6 +1626,7 @@ void smInitConfig(SMXmlConfig_t *smp, SMDPLXmlConfig_t *dplp, SMMcastConfig_t *m DEFAULT_AND_CKSUM_U32(smp->node_appearance_msg_thresh, 0, CKSUM_OVERALL_DISRUPT_CONSIST); DEFAULT_AND_CKSUM_U32(smp->spine_first_routing, 0, CKSUM_OVERALL_DISRUPT_CONSIST); DEFAULT_AND_CKSUM_U32(smp->shortestPathBalanced, 1, CKSUM_OVERALL_DISRUPT_CONSIST); + DEFAULT_AND_CKSUM_U32(smp->hypercube, 0, CKSUM_OVERALL_DISRUPT_CONSIST); DEFAULT_AND_CKSUM_U32(smp->lid, 0x0, CKSUM_OVERALL_DISRUPT); DEFAULT_U32(smp->lmc, 0x0); @@ -1719,6 +1787,24 @@ void smInitConfig(SMXmlConfig_t *smp, SMDPLXmlConfig_t *dplp, SMMcastConfig_t *m CKSUM_STR(smp->dgRouting.dg[i].member, CKSUM_OVERALL_DISRUPT_CONSIST); } + SPRoutingCtrl = smp->SPRoutingCtrl; + if (SPRoutingCtrl == (void *)~0ul) { + SPRoutingCtrl = NULL; + smp->SPRoutingCtrl = NULL; + } + while (SPRoutingCtrl) { + SmSPRoutingPort_t *ports = &SPRoutingCtrl->ports[i]; + int portCount = SPRoutingCtrl->portCount; + + CKSUM_DATA(SPRoutingCtrl->guid, CKSUM_OVERALL_DISRUPT_CONSIST); + for (i = 0; i < portCount; i++, ports++) { + CKSUM_DATA(ports->pport, CKSUM_OVERALL_DISRUPT_CONSIST); + CKSUM_DATA(ports->vport, CKSUM_OVERALL_DISRUPT_CONSIST); + CKSUM_DATA(ports->cost, CKSUM_OVERALL_DISRUPT_CONSIST); + } + SPRoutingCtrl = SPRoutingCtrl->next; + } + #ifdef CONFIG_INCLUDE_DOR if (!strcasecmp(smp->routing_algorithm, "dor-updown")) { // If using dor-updown, checksum the smDorRouting config. @@ -1888,6 +1974,7 @@ void smShowConfig(SMXmlConfig_t *smp, SMDPLXmlConfig_t *dplp, SMMcastConfig_t *m { uint32_t i; char str[32]; + SmSPRoutingCtrl_t *SPRoutingCtrl; if (!smp || !dplp || !mcp || !mlsp) return; @@ -1922,6 +2009,7 @@ void smShowConfig(SMXmlConfig_t *smp, SMDPLXmlConfig_t *dplp, SMMcastConfig_t *m printf("XML - node_appearance_msg_thresh %u\n", (unsigned int)smp->node_appearance_msg_thresh); printf("XML - spine_first_routing %u\n", (unsigned int)smp->spine_first_routing); printf("XML - shortestPathBalanced %u\n", (unsigned int)smp->shortestPathBalanced); + printf("XML - hypercube %u\n", (unsigned int)smp->hypercube); printf("XML - lid 0x%x\n", (unsigned int)smp->lid); printf("XML - lmc 0x%x\n", (unsigned int)smp->lmc); printf("XML - lmc_e0 0x%x\n", (unsigned int)smp->lmc_e0); @@ -2028,6 +2116,17 @@ void smShowConfig(SMXmlConfig_t *smp, SMDPLXmlConfig_t *dplp, SMMcastConfig_t *m printf("XML - ftreeRouting.fis_on_same_tier %u\n", (unsigned int)smp->ftreeRouting.fis_on_same_tier); printf("XML - ftreeRouting.coreSwitches %s\n", smp->ftreeRouting.coreSwitches.member); printf("XML - ftreeRouting.routeLast %s\n", smp->ftreeRouting.routeLast.member); + SPRoutingCtrl = smp->SPRoutingCtrl; + while (SPRoutingCtrl) { + SmSPRoutingPort_t *ports = SPRoutingCtrl->ports; + int portCount = SPRoutingCtrl->portCount; + + printf("XML - SPRoutingCtrl.guid = 0x016%"PRIx64", pPort, vPort, cost:", SPRoutingCtrl->guid); + for (i = 0; i < portCount; i++, ports++) { + printf("XML %d %d %d\n", ports->pport, ports->vport, ports->cost); + } + SPRoutingCtrl = SPRoutingCtrl->next; + } printf("XML - cableInfoPolicy %u\n", (unsigned int)smp->cableInfoPolicy); printf("XML - terminateAfter %u\n", (unsigned int)smp->terminateAfter); printf("XML - dumpCounters %s\n", smp->dumpCounters); @@ -5874,6 +5973,127 @@ static IXML_FIELD SmDGRoutingFields[] = { { NULL } }; +static IXML_FIELD XmlSPRoutingPortFields[] = { + { tag:"pPort", format:'u', IXML_FIELD_INFO(SmSPRoutingPort_t, pport) }, + { tag:"vPort", format:'u', IXML_FIELD_INFO(SmSPRoutingPort_t, vport) }, + { tag:"Cost", format:'u', IXML_FIELD_INFO(SmSPRoutingPort_t, cost) }, + { NULL } +}; + +static void XmlSPRoutingPortParserEnd(IXmlParserState_t *state, const IXML_FIELD *field, void *object, void *parent, XML_Char *content, unsigned len, boolean valid) +{ + SmSPRoutingPort_t *PortData = object; + uint8_t pport = PortData->pport; + uint8_t vport = PortData->vport; + uint16_t cost = PortData->cost; + SmSPRoutingCtrl_t *SPRoutingCtrl = parent; + portMap_t *pportMap = SPRoutingCtrl->pportMap; + portMap_t *vportMap = SPRoutingCtrl->vportMap; + SmSPRoutingPort_t *ports = SPRoutingCtrl->ports; + uint16_t portCount = SPRoutingCtrl->portCount; + uint16_t portsSize = ROUNDUP(portCount, PORTS_ALLOC_UNIT); + + if (xml_parse_debug) { + fprintf(stdout, "%s %s instance %u common %u\n", __FUNCTION__, + field->tag, (unsigned int)instance, (unsigned int)common); + } + + if (!valid) { + fprintf(stderr, "Error processing XML %s tag\n", field->tag); + return; + } + + if (pport >= STL_MAX_PORTS) { + IXmlParserPrintError(state, "bad pPort %d", pport); + return; + } + if (vport >= STL_MAX_PORTS) { + IXmlParserPrintError(state, "bad vPort %d", vport); + return; + } + + // check for duplicates + + if (pport) { + if (portMapTest(pportMap, pport)) { + IXmlParserPrintError(state, "pPort %d already specified", pport); + return; + } + portMapSet(pportMap, pport); + } + + if (vport) { + if (portMapTest(vportMap, vport)) { + IXmlParserPrintError(state, "vPort %d already specified", vport); + return; + } + portMapSet(vportMap, vport); + } + + if (portCount >= portsSize) { + portsSize += PORTS_ALLOC_UNIT; + ports = getXmlMemory(portsSize*sizeof(*ports), "SmSPRoutingPort_t *ports"); + if (!ports) { + PRINT_MEMORY_ERROR; + return; + } + if (portCount) { + memcpy(ports, SPRoutingCtrl->ports, portCount*sizeof(*ports)); + freeXmlMemory(SPRoutingCtrl->ports, portCount*sizeof(*ports), + "SmSPRoutingPort_t *ports"); + } + SPRoutingCtrl->ports = ports; + } + ports += portCount; + ports->pport = pport; + ports->vport = vport; + ports->cost = cost; + SPRoutingCtrl->portCount++; +}; + +static IXML_FIELD XmlSPRoutingCtrlFields[] = { + { tag:"SwitchGuid", format:'x', IXML_FIELD_INFO(SmSPRoutingCtrl_t, guid) }, + { tag:"PortData", format:'k', subfields:XmlSPRoutingPortFields, start_func:IXmlParserStartStruct, end_func:XmlSPRoutingPortParserEnd }, + { NULL } +}; + +static void XmlSPRoutingCtrlParserEnd(IXmlParserState_t *state, const IXML_FIELD *field, void *object, void *parent, XML_Char *content, unsigned len, boolean valid) +{ + SmSPRoutingCtrl_t *XmlSPRoutingCtrl = object; + uint64_t guid = XmlSPRoutingCtrl->guid; + SMXmlConfig_t *smp = (SMXmlConfig_t*)parent; + SmSPRoutingCtrl_t **prev = &smp->SPRoutingCtrl; + SmSPRoutingCtrl_t *SPRoutingCtrl = smp->SPRoutingCtrl; + + if (xml_parse_debug) { + fprintf(stdout, "%s %s instance %u common %u\n", __FUNCTION__, + field->tag, (unsigned int)instance, (unsigned int)common); + } + + if (!valid) { + fprintf(stderr, "Error processing XML %s tag\n", field->tag); + return; + } + + // see if we already have this GUID + while (SPRoutingCtrl) { + if (SPRoutingCtrl->guid == guid) { + IXmlParserPrintError(state, "guid 0x%016"PRIx64" already entered", + guid); + return; + } + prev = &SPRoutingCtrl->next; + SPRoutingCtrl = *prev; + } + SPRoutingCtrl = getXmlMemory(sizeof(*SPRoutingCtrl), "SmSPRoutingCtrl_t *SPRoutingCtrl"); + if (!SPRoutingCtrl) { + PRINT_MEMORY_ERROR; + return; + } + *SPRoutingCtrl = *XmlSPRoutingCtrl; + *prev = SPRoutingCtrl; +} + // "Sm/AdaptiveRouting" start tag static void* SmAdaptiveRoutingXmlParserStart(IXmlParserState_t *state, void *parent, const char **attr) { @@ -5981,6 +6201,7 @@ static IXML_FIELD SmFields[] = { { tag:"NodeAppearanceMsgThreshold", format:'u', IXML_FIELD_INFO(SMXmlConfig_t, node_appearance_msg_thresh) }, { tag:"SpineFirstRouting", format:'u', IXML_FIELD_INFO(SMXmlConfig_t, spine_first_routing) }, { tag:"ShortestPathBalanced", format:'u', IXML_FIELD_INFO(SMXmlConfig_t, shortestPathBalanced) }, + { tag:"Hypercube", format:'u', IXML_FIELD_INFO(SMXmlConfig_t, hypercube) }, { tag:"PathSelection", format:'u', IXML_FIELD_INFO(SMXmlConfig_t, path_selection), end_func:SmPathSelectionParserEnd }, { tag:"QueryValidation", format:'u', IXML_FIELD_INFO(SMXmlConfig_t, queryValidation) }, { tag:"SmaBatchSize", format:'u', IXML_FIELD_INFO(SMXmlConfig_t, sma_batch_size) }, @@ -6050,6 +6271,7 @@ static IXML_FIELD SmFields[] = { { tag:"AdaptiveRouting", format:'k', subfields:SmAdaptiveRoutingFields, start_func:SmAdaptiveRoutingXmlParserStart, end_func:SmAdaptiveRoutingXmlParserEnd }, { tag:"FatTreeTopology", format:'k', subfields:SmFtreeRoutingFields, start_func:SmFtreeRoutingXmlParserStart, end_func:SmFtreeRoutingXmlParserEnd }, { tag:"DGShortestPathTopology", format:'k', subfields:SmDGRoutingFields, start_func:SmDGRoutingXmlParserStart, end_func:SmDGRoutingXmlParserEnd }, + { tag:"SPRoutingCtrl", format:'k', size:sizeof(SmSPRoutingCtrl_t), subfields:XmlSPRoutingCtrlFields, start_func:IXmlParserStartStruct, end_func:XmlSPRoutingCtrlParserEnd }, #ifdef CONFIG_INCLUDE_DOR { tag:"MeshTorusTopology", format:'k', subfields:SmDorRoutingFields, start_func:SmDorRoutingXmlParserStart, end_func:SmDorRoutingXmlParserEnd }, #endif diff --git a/Esm/ib/src/linux/startup/opafm_src.xml b/Esm/ib/src/linux/startup/opafm_src.xml index 7c0141de..f2b8db19 100644 --- a/Esm/ib/src/linux/startup/opafm_src.xml +++ b/Esm/ib/src/linux/startup/opafm_src.xml @@ -942,6 +942,50 @@ shortestpath + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Esm/ib/src/smi/include/sm_l.h b/Esm/ib/src/smi/include/sm_l.h index 5fc88e16..48d876b6 100755 --- a/Esm/ib/src/smi/include/sm_l.h +++ b/Esm/ib/src/smi/include/sm_l.h @@ -323,6 +323,7 @@ typedef struct _PortData { } changes; void *routingData; // Private data used by topology algorithm. + uint16_t routingCost; int32_t initWireDepth; // Initial wire depth to use for buffer calculations. } PortData_t; @@ -369,6 +370,7 @@ typedef struct _Node { PORT *pgft; ///< Port Group Forwarding Table uint32_t pgftSize; ///< amount of memory allocated for pgft; should track actual memory allocation, not highest entry in use Port_t *port; // ports on this node + uint8_t *portOrder; bitset_t activePorts; bitset_t initPorts; bitset_t vfMember; @@ -1196,6 +1198,9 @@ void Switch_Enqueue_Type(Topology_t *, Node_t *, int, int); if (NODEP->pgft) { \ sm_Node_release_pgft(NODEP); \ }\ + if (NODEP->portOrder) { \ + vs_pool_free(&sm_pool, NODEP->portOrder); \ + }\ if (NODEP->nodeDescString) { \ vs_pool_free(&sm_pool, NODEP->nodeDescString); \ }\ @@ -1517,7 +1522,7 @@ static __inline__ Lid_t sm_port_top_lid(Port_t * portp) { (((Z) == STL_LINK_SPEED_12_5G) ? 12500000000ull : 25781250000ull) // Cost should be evenly divisable by (width * SpeedFactor) -#define SpeedWidth_to_Cost(X) ((X) ? 2400/(X) : 2400) +#define SpeedWidth_to_Cost(X) (((X) && !sm_config.hypercube) ? 2400/(X) : 2400) static __inline__ uint8_t sm_GetLsf(STL_PORT_INFO *portInfo) { return (Decode_SpeedFactor(portInfo->LinkSpeed.Active)); @@ -1535,7 +1540,7 @@ static __inline__ uint16_t sm_GetSpeed(PortData_t *portData) { } static __inline__ uint16_t sm_GetCost(PortData_t *portData) { - return (SpeedWidth_to_Cost(sm_GetSpeed(portData))); + return (portData->routingCost ? portData->routingCost : SpeedWidth_to_Cost(sm_GetSpeed(portData))); } static __inline__ char* sm_nodeDescString(Node_t *nodep) { diff --git a/Esm/ib/src/smi/sm/sm_routing.c b/Esm/ib/src/smi/sm/sm_routing.c index a7c98477..2512a394 100644 --- a/Esm/ib/src/smi/sm/sm_routing.c +++ b/Esm/ib/src/smi/sm/sm_routing.c @@ -214,7 +214,7 @@ sm_routing_alloc_floyds(Topology_t *topop) void sm_routing_init_floyds(Topology_t *topop) { - int i, j, k, ij, ik, ki, iNumNodes; + int i, j, k, ij, ik, iNumNodes; Node_t *nodep, *neighborNodep; Port_t *portp; @@ -244,8 +244,7 @@ sm_routing_init_floyds(Topology_t *topop) } ik = Index(i, k); - ki = Index(k, i); - topop->cost[ik] = topop->cost[ki] = sm_GetCost(portp->portData); + topop->cost[ik] = sm_GetCost(portp->portData); } } } @@ -285,13 +284,12 @@ sm_routing_calc_floyds(int switches, unsigned short * cost) continue; } - for (j = i + 1, ij = iNumNodes + i + 1, kj = kNumNodes + i + 1; - j < switches; ++j, ++ij, ++kj) { + for (j = 0, ij = iNumNodes, kj = kNumNodes; + j < switches; + ++j, ++ij, ++kj) { if ((value = cost[ik] + cost[kj]) < cost[ij]) { cost[ij] = value; - // Take advantage of the fact that cost array is symmetric - cost[Index(j, i)] = value; } } } diff --git a/Esm/ib/src/smi/sm/sm_shortestpath.c b/Esm/ib/src/smi/sm/sm_shortestpath.c index 84ee41b4..de930ddb 100644 --- a/Esm/ib/src/smi/sm/sm_shortestpath.c +++ b/Esm/ib/src/smi/sm/sm_shortestpath.c @@ -230,8 +230,11 @@ _select_ports(Topology_t *topop, Node_t *switchp, int endIndex, SwitchportToNext uint16_t best_lidsRouted = 0xffff; uint32_t best_switchLidsRouted = 0xffffffff; int end_port = 0; + int port; Node_t *next_nodep; + Node_t *first_nodep = 0; Port_t *portp; + uint8_t *portOrder = switchp->portOrder; SpineFirstState_t sfstate; SpineFirstResult_t sfres; @@ -240,7 +243,8 @@ _select_ports(Topology_t *topop, Node_t *switchp, int endIndex, SwitchportToNext best_cost = topop->cost[Index(i, j)]; _spine_first_reset(&sfstate); - for_all_physical_ports(switchp, portp) { + for (port = 0; port < switchp->nodeInfo.NumPorts; port++) { + portp = sm_get_port(switchp, portOrder ? portOrder[port] : port + 1); if (!sm_valid_port(portp) || portp->state <= IB_PORT_DOWN) continue; next_nodep = sm_find_node(topop, portp->nodeno); @@ -284,9 +288,13 @@ _select_ports(Topology_t *topop, Node_t *switchp, int endIndex, SwitchportToNext best_speed = cur_speed; best_lidsRouted = portp->portData->lidsRouted; best_switchLidsRouted = next_nodep->numLidsRouted; + first_nodep = next_nodep; end_port = 0; } else if (selectBest) { + if ((sm_config.hypercube) && (next_nodep != first_nodep)) { + break; + } if (portp->portData->lidsRouted < best_lidsRouted) { best_lidsRouted = portp->portData->lidsRouted; best_switchLidsRouted = next_nodep->numLidsRouted; diff --git a/Esm/ib/src/smi/sm/sm_utility.c b/Esm/ib/src/smi/sm/sm_utility.c index c4eb2230..7c068b67 100755 --- a/Esm/ib/src/smi/sm/sm_utility.c +++ b/Esm/ib/src/smi/sm/sm_utility.c @@ -2294,6 +2294,152 @@ sm_validate_predef_fields(Topology_t* topop, FabricData_t* pdtop, Node_t * cnp, } +static void +sm_setup_routing_ctrl(Node_t *nodep) +{ + STL_NODE_INFO *nodeInfo = &nodep->nodeInfo; + uint64_t NodeGUID = nodeInfo->NodeGUID; + SmSPRoutingCtrl_t *guidData; + SmSPRoutingCtrl_t *defaultData = NULL; + SmSPRoutingCtrl_t *nodeData = NULL; + SmSPRoutingPort_t *ports; + uint16_t portCount; + uint16_t defaultCost = 0; + int NumPorts = nodeInfo->NumPorts; + Port_t *portp; + portMap_t pportMap[STL_MAX_PORTS/(sizeof(portMap_t)*8) + 1]; + portMap_t vportMap[STL_MAX_PORTS/(sizeof(portMap_t)*8) + 1]; + int i; + int pport; + int vport; + uint8_t *portOrder; + Status_t status; + uint16_t cost; + + if (nodeInfo->NodeType != NI_TYPE_SWITCH) { + return; + } + + if (nodep->portOrder) { + return; + } + + for (guidData = sm_config.SPRoutingCtrl; guidData; guidData = guidData->next) { + if (!guidData->guid) { + defaultData = guidData; + } else if (guidData->guid == NodeGUID) { + nodeData = guidData; + } + } + guidData = nodeData ? nodeData : defaultData; + if (!guidData) { + // nothing to do + return; + } + + if (nodeData) { + // look for a default cost within the node's entry + ports = nodeData->ports; + portCount = nodeData->portCount; + for (i = 0; i < portCount; i++, ports++) { + if (!ports->pport) { + defaultCost = ports->cost; + break; + } + } + } + if (!defaultCost && defaultData) { + // the node didn't have a default cost, look for a global one + ports = defaultData->ports; + portCount = defaultData->portCount; + for (i = 0; i < portCount; i++, ports++) { + if (!ports->pport) { + defaultCost = ports->cost; + break; + } + } + } + + if (defaultCost) { + // pre-initialize cost to the default cost if there is one + for_all_ports(nodep, portp) { + if (portp->portData) { + portp->portData->routingCost = defaultCost; + } + } + } + + status = vs_pool_alloc(&sm_pool, NumPorts*sizeof(*portOrder), + (void *)&portOrder); + if (status != VSTATUS_OK) { + IB_LOG_ERROR_FMT(__func__, "can't malloc portOrder status=%d", + status); + return; + } + nodep->portOrder = portOrder; + + memcpy(pportMap, guidData->pportMap, sizeof(pportMap)); + memcpy(vportMap, guidData->vportMap, sizeof(vportMap)); + + // process the node's entries + ports = guidData->ports; + portCount = guidData->portCount; + for (i = 0; i < portCount; i++, ports++) { + pport = ports->pport; + vport = ports->vport; + cost = ports->cost; + if (cost) { + portp = sm_get_port(nodep, pport); + if (portp && portp->portData) { + portp->portData->routingCost = cost; + } + } + if (vport && (vport <= NumPorts) && pport && (pport <= NumPorts)) { + portOrder[vport] = pport; + } + } + + // add in port order entries from default if not already there + if (defaultData && nodeData) { + ports = defaultData->ports; + portCount = defaultData->portCount; + for (i = 0; i < portCount; i++, ports++) { + pport = ports->pport; + vport = ports->vport; + cost = ports->cost; + if (cost) { + portp = sm_get_port(nodep, pport); + if (portp && portp->portData) { + portp->portData->routingCost = cost; + } + } + if (portMapTest(pportMap, pport) || portMapTest(vportMap, vport)) { + continue; + } + if (vport && (vport <= NumPorts) && pport && (pport <= NumPorts)) { + portOrder[vport] = pport; + portMapSet(pportMap, pport); + portMapSet(vportMap, vport); + } + } + } + + pport = 1; + for (vport = 1; vport <= NumPorts; vport++) { + if (!portMapTest(vportMap, vport)) { + while (pport <= NumPorts) { + if (!portMapTest(pportMap, pport)) { + portOrder[vport] = pport++; + break; + } + pport++; + } + } + } + return; +} + + // the special case of cnp==NULL and cpp==NULL only occurs when SM is trying its // its own port, for all other nodes in fabric these will both be supplied Status_t @@ -3253,6 +3399,9 @@ sm_setup_node(Topology_t * topop, FabricData_t * pdtop, Node_t * cnp, Port_t * c // initialize VFs for the node smSetupNodeVFs(nodep); + + // setup the routing control data + sm_setup_routing_ctrl(nodep); } nodep->aggregateEnable = sm_config.use_aggregates;