Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix for Issue#100 #4

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
154 changes: 151 additions & 3 deletions aci-preupgrade-validation-script.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,6 @@ def connect(self):
logging.debug(
"spawning new pexpect connection: ssh %s@%s -p %d" % (self.username, self.hostname, self.port))
no_verify = " -o StrictHostKeyChecking=no -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null"
no_verify += " -o HostKeyAlgorithms=+ssh-dss"
if self.verify: no_verify = ""
self.child = pexpect.spawn("ssh %s %s@%s -p %d" % (no_verify, self.username, self.hostname, self.port),
searchwindowsize=self.searchwindowsize)
Expand Down Expand Up @@ -3418,6 +3417,7 @@ def subnet_scope_check(index, total_checks, cversion, **kwargs):
print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)
return result


def rtmap_comm_match_defect_check(index, total_checks, tversion, **kwargs):
title = 'Route-map Community Match Defect'
result = PASS
Expand Down Expand Up @@ -3492,10 +3492,11 @@ def invalid_fex_rs_check(index, total_checks, **kwargs):

for rs in infraRsHPathAtt:
dn = rs["infraRsHPathAtt"]["attributes"]["dn"]
m = re.search(r'eth(?P<fex>\d+)\/\d\/\d', dn)
m = re.search(r'eth(?P<fex>\d{3})\/\d\/\d', dn)
if m:
fex_id = m.group('fex')
data.append([fex_id, dn])
if int(fex_id) >= 101:
data.append([fex_id, dn])

if data:
result = FAIL_UF
Expand Down Expand Up @@ -3530,6 +3531,150 @@ def lldp_custom_int_description_defect_check(index, total_checks, tversion, **kw
return result


def unsupported_fec_configuration_ex_check(index, total_checks, sw_cversion, tversion, **kwargs):
title = 'Unsupported FEC Configuration For N9K-C93180YC-EX'
result = PASS
msg = ''
headers = ["Pod ID", "Node ID", "Switch Model", "Interface", "FEC Mode"]
data = []
recommended_action = 'Nexus C93180YC-EX switches do not support IEEE-RS-FEC or CONS16-RS-FEC mode. Misconfigured ports will be hardware disabled upon upgrade. Remove unsupported FEC configuration prior to upgrade.'
doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#unsupported-fec-configuration-for-n9k-c93180yc-ex'
print_title(title, index, total_checks)

if not tversion:
print_result(title, MANUAL, "Target version not supplied. Skipping.")
return MANUAL

if sw_cversion.older_than('5.0(1a)') and tversion.newer_than("5.0(1a)"):
api = 'topSystem.json'
api += '?rsp-subtree=children&rsp-subtree-class=l1PhysIf,eqptCh'
api += '&rsp-subtree-filter=or(eq(l1PhysIf.fecMode,"ieee-rs-fec"),eq(l1PhysIf.fecMode,"cons16-rs-fec"),eq(eqptCh.model,"N9K-C93180YC-EX"))'
api += '&rsp-subtree-include=required'
topSystems = icurl('class', api)
for topSystem in topSystems:
model = None
l1PhysIfs = []
for child in topSystem['topSystem']['children']:
if child.get("eqptCh"):
model = child['eqptCh']['attributes']['model']
elif child.get("l1PhysIf"):
interface = child['l1PhysIf']['attributes']['id']
fecMode = child['l1PhysIf']['attributes']['fecMode']
l1PhysIfs.append({"interface":interface,"fecMode":fecMode})
if model and l1PhysIfs:
pod_id = topSystem['topSystem']['attributes']['podId']
node_id = topSystem['topSystem']['attributes']['id']
for l1PhysIf in l1PhysIfs:
data.append([pod_id,node_id,model,l1PhysIf['interface'],l1PhysIf['fecMode']])
if data:
result = FAIL_O

print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)
return result


def static_route_overlap_check(index, total_checks, cversion, tversion, **kwargs):
title = 'L3out /32 Static Route and BD Subnet Overlap'
result = PASS
msg = ''
headers = ['L3out', '/32 Static Route', 'BD', 'BD Subnet']
data = []
recommended_action = 'Change /32 static route design or target a fixed version'
doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#l3out-32-overlap-with-bd-subnet'
print_title(title, index, total_checks)
iproute_regex = r'uni/tn-(?P<tenant>[^/]+)/out-(?P<l3out>[^/]+)/lnodep-(?P<nodeprofile>[^/]+)/rsnodeL3OutAtt-\[topology/pod-(?P<pod>[^/]+)/node-(?P<node>\d{3,4})\]/rt-\[(?P<addr>[^/]+)/(?P<netmask>\d{1,2})\]'
# bd_regex = r'uni/tn-(?P<tenant>[^/]+)/BD-(?P<bd>[^/]+)/rsctx'
bd_subnet_regex = r'uni/tn-(?P<tenant>[^/]+)/BD-(?P<bd>[^/]+)/subnet-\[(?P<subnet>[^/]+/\d{2})\]'

if (cversion.older_than("5.2(6e)") and tversion.newer_than("5.0(1a)") and tversion.older_than("5.2(6e)") ):
slash32filter = 'ipRouteP.json?query-target-filter=and(wcard(ipRouteP.dn,"/32"))'
staticRoutes = icurl('class', slash32filter)
if staticRoutes:
staticroute_vrf = icurl('class', 'l3extRsEctx.json')
staticR_to_vrf = {}
for staticRoute in staticRoutes:
staticroute_array = re.search(iproute_regex, staticRoute['ipRouteP']['attributes']['dn'])
l3out_dn = 'uni/tn-' + staticroute_array.group("tenant") + '/out-' + staticroute_array.group("l3out")+ '/rsectx'

for l3outCtx in staticroute_vrf:
l3outCtx_Vrf = {}
if l3outCtx['l3extRsEctx']['attributes']['dn'] == l3out_dn:
l3outCtx_Vrf['vrf'] = l3outCtx['l3extRsEctx']['attributes']['tDn']
l3outCtx_Vrf['l3out'] = l3outCtx['l3extRsEctx']['attributes']['dn'].replace('/rsectx', '')
staticR_to_vrf[staticroute_array.group("addr")] = l3outCtx_Vrf


bds_in_vrf = icurl('class', 'fvRsCtx.json')
vrf_to_bd = {}
for bd_ref in bds_in_vrf:
vrf_name = bd_ref['fvRsCtx']['attributes']['tDn']
bd_list = vrf_to_bd.get(vrf_name, [])
bd_name = bd_ref['fvRsCtx']['attributes']['dn'].replace('/rsctx','')
bd_list.append(bd_name)
vrf_to_bd[vrf_name] = bd_list

subnets_in_bd = icurl('class', 'fvSubnet.json')
bd_to_subnet = {}
for subnet in subnets_in_bd:
bd_subnet_re = re.search(bd_subnet_regex, subnet['fvSubnet']['attributes']['dn'])
if bd_subnet_re:
bd_dn = 'uni/tn-' + bd_subnet_re.group("tenant") + '/BD-' + bd_subnet_re.group("bd")
subnet_list = bd_to_subnet.get(bd_dn, [])
subnet_list.append(bd_subnet_re.group("subnet"))
bd_to_subnet[bd_dn] = subnet_list

for static_route, info in staticR_to_vrf.items():
for bd in vrf_to_bd[info['vrf']]:
for subnet in bd_to_subnet[bd]:
if IPAddress.ip_in_subnet(static_route, subnet):
data.append([info['l3out'], static_route, bd, subnet])

if data:
result = FAIL_O

print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)
return result


def validate_32_64_bit_image_check(index, total_checks, tversion, **kwargs):
title = '32 and 64-Bit Firmware Image for Switches'
result = PASS
msg = ''
headers = ["Target Switch Version", "32-Bit Image Found", "64-Bit Image Found"]
data = []
recommended_action = 'Upload the missing 32 or 64 bit Switch Image to the Firmware repository'
doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#602-requires-32-and-64-bit-switch-images'
print_title(title, index, total_checks)

if not tversion:
print_result(title, MANUAL, "Target version not supplied. Skipping.")
return MANUAL

if tversion.newer_than("6.0(2a)"):
found32 = found64 = False
target_sw_ver = 'n9000-1' + tversion.version
firmware_api = 'firmwareFirmware.json'
firmware_api += '?query-target-filter=eq(firmwareFirmware.fullVersion,"%s")' % (target_sw_ver)
firmwares = icurl('class', firmware_api)

for firmware in firmwares:
if firmware['firmwareFirmware']['attributes']['bitInfo'] == '32':
found32 = True
elif firmware['firmwareFirmware']['attributes']['bitInfo'] == '64':
found64 = True

if not found32 or not found64:
result = FAIL_UF
data.append([target_sw_ver, found32, found64])

else:
result = NA
msg = 'Target version below 6.0(2)'

print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)
return result


if __name__ == "__main__":
prints(' ==== %s%s, Script Version %s ====\n' % (ts, tz, SCRIPT_VERSION))
prints('!!!! Check https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script for Latest Release !!!!\n')
Expand Down Expand Up @@ -3569,6 +3714,7 @@ def lldp_custom_int_description_defect_check(index, total_checks, tversion, **kw
switch_group_guideline_check,
mini_aci_6_0_2_check,
post_upgrade_cb_check,
validate_32_64_bit_image_check,

# Faults
apic_disk_space_faults_check,
Expand Down Expand Up @@ -3608,6 +3754,7 @@ def lldp_custom_int_description_defect_check(index, total_checks, tversion, **kw
oob_mgmt_security_check,
eecdh_cipher_check,
subnet_scope_check,
unsupported_fec_configuration_ex_check,

# Bugs
ep_announce_check,
Expand All @@ -3626,6 +3773,7 @@ def lldp_custom_int_description_defect_check(index, total_checks, tversion, **kw
invalid_fex_rs_check,
lldp_custom_int_description_defect_check,
rtmap_comm_match_defect_check,
static_route_overlap_check

]
summary = {PASS: 0, FAIL_O: 0, FAIL_UF: 0, ERROR: 0, MANUAL: 0, POST: 0, NA: 0, 'TOTAL': len(checks)}
Expand Down
61 changes: 58 additions & 3 deletions docs/docs/validations.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ Items | This Script
[Intersight Device Connector upgrade status][g13] | :white_check_mark: | :white_check_mark: 4.2(5) | :white_check_mark:
[Mini ACI Upgrade to 6.0(2)+][g14] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:
[Post Upgrade CallBack Integrity][g15] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:
[6.0(2)+ requires 32 and 64 bit switch images][g16] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:


[g1]: #compatibility-target-aci-version
Expand All @@ -50,7 +51,7 @@ Items | This Script
[g13]: #intersight-device-connector-upgrade-status
[g14]: #mini-aci-upgrade-to-602-or-later
[g15]: #post-upgrade-callback-integrity

[g16]: #602-requires-32-and-64-bit-switch-images

### Fault Checks
Items | Faults | This Script | APIC built-in | Pre-Upgrade Validator (App)
Expand Down Expand Up @@ -116,6 +117,7 @@ Items | Faults | This Script
[OoB Mgmt Security][c13] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:
[EECDH SSL Cipher Disabled][c14] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:
[BD and EPG Subnet must have matching scopes][c15] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:
[Unsupported FEC Configuration for N9K-C93180YC-EX][c16] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:


[c1]: #vpc-paired-leaf-switches
Expand All @@ -133,6 +135,7 @@ Items | Faults | This Script
[c13]: #oob-mgmt-security
[c14]: #eecdh-ssl-cipher
[c15]: #bd-and-epg-subnet-must-have-matching-scopes
[c16]: #unsupported-fec-configuration-for-n9k-c93180yc-ex


### Defect Condition Checks
Expand All @@ -155,6 +158,8 @@ Items | Defect | This Script
[Invalid FEX fabricPathEp DN References][d14] | CSCwh68103 | :white_check_mark: | :no_entry_sign: |:no_entry_sign:
[LLDP Custom Interface Description][d15] | CSCwf00416 | :white_check_mark: | :no_entry_sign: |:no_entry_sign:
[Route-map Community Match][d16] | CSCwb08081 | :white_check_mark: | :no_entry_sign: |:no_entry_sign:
[L3out /32 overlap with BD Subnet][d17] | CSCwb91766 | :white_check_mark: | :no_entry_sign: |:no_entry_sign:


[d1]: #ep-announce-compatibility
[d2]: #eventmgr-db-size-defect-susceptibility
Expand All @@ -172,7 +177,7 @@ Items | Defect | This Script
[d14]: #invalid-fex-fabricpathep-dn-references
[d15]: #lldp-custom-interface-description
[d16]: #route-map-community-match

[d17]: #l3out-32-overlap-with-bd-subnet


## General Check Details
Expand Down Expand Up @@ -412,6 +417,14 @@ This validation checks whether the number of objects for the existing and newly
status :
```


### 6.0(2)+ requires 32 and 64 bit switch images

When targeting any version that is 6.0(2) or greater, download both the 32-bit and 64-bit Cisco ACI-mode switch images to the Cisco APIC. Downloading only one of the images may result in errors during the upgrade process.

For additional information, see the [Guidelines and Limitations for Upgrading or Downgrading][28] section of the Cisco APIC Installation and ACI Upgrade and Downgrade Guide.


## Fault Check Details

### APIC Disk Space Usage
Expand Down Expand Up @@ -1641,6 +1654,38 @@ Before the fix implemented in [CSCvv30303][21], it was possible for a BD and an

[CSCvv30303][21] introduced policy validations to prevent this configuration and enforce that matching subnets defined in both the BD and related EPG have the same scope. It is imperative that identified mismatching subnet scopes are corrected within an ACI fabric to prevent unexpected traffic pattern issues in the future.

### Unsupported FEC Configuration for N9K-C93180YC-EX

Nexus C93180YC-EX switches [do not support CONS16-RS-FEC or IEEE-RS-FEC][26] mode. In older versions a FEC mode misconfiguration does not cause the interface to be down. However after upgrading to 5.0(1) or later the port will become hardware disabled until the misconfiguration is removed.

It is important to remove any unsupported configuration prior to ugprade to avoid any unexpected downtime.

!!! example
On each N9K-C93180YC-EX switch, you can check the FEC mode for each interface with the following query:
```
leaf101# moquery -c l1PhysIf -x query-target-filter=or\(eq\(l1PhysIf.fecMode,\"ieee-rs-fec\"\),eq\(l1PhysIf.fecMode,\"cons16-rs-fec\"\)\)
Total Objects shown: 1

# l1.PhysIf
id : eth1/1
adminSt : up
autoNeg : on
breakT : nonbroken
bw : 0
childAction :
delay : 1
descr :
dfeDelayMs : 0
dn : sys/phys-[eth1/1]
dot1qEtherType : 0x8100
emiRetrain : disable
enablePoap : no
ethpmCfgFailedBmp :
ethpmCfgFailedTs : 00:00:00:00.000
ethpmCfgState : 2
fcotChannelNumber : Channel32
fecMode : ieee-rs-fec <<<
```

## Defect Check Details

Expand Down Expand Up @@ -1807,7 +1852,6 @@ The problem is related only to the front-panel interfaces Ethernet 1/1- 1/48. Op
Because of this, the target version of your upgrade must be a version with a fix of CSCwh81430 when your fabric includes those switches mentioned above. See the Field Notice [FN74085][20] for details.



### Invalid FEX fabricPathEp DN References

If you have deployed a FEX on a version prior to having validations introduced in [CSCwh68103][23], it is possible that `fabricPathEp` objects were created with an incorrect DN format. As a result, the related `infraRsHPathAtt` objects pointing to those `fabricPathEp` will also contain the invalid DN in their DN formatting given how ACI builds out object relations.
Expand Down Expand Up @@ -1884,6 +1928,14 @@ It is recommended if you are upgrading to an affected release to add a prefix li
dn : uni/tn-Cisco/subj-match-comm/dest-[0.0.0.0/0]
```


### L3out /32 overlap with BD Subnet

Due to defect [CSCwb91766][27], L3out /32 Static Routes that overlap with BD Subnets within the same VRF will be programmed into RIB but not FIB of the relevant switches in the forwarding path. This will cause traffic loss as packets will not be able to take the /32 route, resulting in unexpecteded forwarding issues.

If found, the target version of your upgrade should be a version with a fix for CSCwb91766. Otherwise, the other option is to change the routing design of the affected fabric.


[0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
[1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
[2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html
Expand All @@ -1910,3 +1962,6 @@ It is recommended if you are upgrading to an affected release to add a prefix li
[23]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwh68103
[24]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwf00416
[25]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwb08081
[26]: https://www.cisco.com/c/en/us/td/docs/switches/datacenter/aci/apic/sw/kb/b_Cisco_ACI_and_Forward_Error_Correction.html#Cisco_Reference.dita_5cef69b3-b7fa-4bde-ba60-38129c9e7d82
[27]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwb91766
[28]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-aci-upgrade-downgrade-architecture.html#Cisco_Reference.dita_22480abb-4138-416b-8dd5-ecde23f707b4
44 changes: 44 additions & 0 deletions tests/invalid_fex_rs_check/infraRsHPathAtt_neg.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,49 @@
"userdom": ""
}
}
},
{
"infraRsHPathAtt": {
"attributes": {
"annotation": "",
"childAction": "",
"dn": "uni/infra/hpaths-105_eth198_1_7/rsHPathAtt-[topology/pod-1/paths-105/pathep-[eth100/1/7]]",
"extMngdBy": "",
"forceResolve": "yes",
"lcOwn": "local",
"modTs": "2021-10-23T08:33:12.722+09:00",
"rType": "mo",
"state": "unformed",
"stateQual": "none",
"status": "",
"tCl": "fabricPathEp",
"tDn": "topology/pod-1/paths-105/pathep-[eth198/1/7]",
"tType": "mo",
"uid": "15374",
"userdom": ""
}
}
},
{
"infraRsHPathAtt": {
"attributes": {
"annotation": "",
"childAction": "",
"dn": "uni/infra/hpaths-105_eth198_1_7/rsHPathAtt-[topology/pod-1/paths-105/pathep-[eth1/30/11]]",
"extMngdBy": "",
"forceResolve": "yes",
"lcOwn": "local",
"modTs": "2021-10-23T08:33:12.722+09:00",
"rType": "mo",
"state": "unformed",
"stateQual": "none",
"status": "",
"tCl": "fabricPathEp",
"tDn": "topology/pod-1/paths-105/pathep-[eth198/1/7]",
"tType": "mo",
"uid": "15374",
"userdom": ""
}
}
}
]
Loading