Added new script validation for CSCwn37676

aci-preupgrade-validation-script.py

@@ -5962,6 +5962,107 @@ def configpush_shard_check(tversion, **kwargs):
 
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
+@check_wrapper(check_title = 'Bootx Service failure log & firmware/tmp directory checks')
+def bootx_firmware_tmp_check(fabric_nodes, cversion, username, password, **kwargs):
+    result = PASS
+    headers = ["Node", "File Count", "Fatal Errors Found", "Status"]
+    data = []
+    recommended_action = 'Contact Cisco TAC to investigate all flagged high file and log counts'
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#bootx_service_failure_log_and_firmware_tmp_directory_checks'
+
+    if not cversion:
+        return Result(result=MANUAL, msg="Current version not provided")
+
+
+    affected = False
+    if (not cversion.older_than("6.0(2h)") and not cversion.newer_than("6.0(8h)")) or \
+       (not cversion.older_than("6.1(1f)") and not cversion.newer_than("6.1(2g)")):
+        affected = True
+
+    if not affected:
+        return Result(result=PASS, msg=VER_NOT_AFFECTED)
+
+    if not fabric_nodes:
+        return Result(result=ERROR, msg="Fabric node response empty. Is the cluster healthy?", doc_url=doc_url)
+
+    # Filter for controller nodes only
+    controller = [node for node in fabric_nodes if node['fabricNode']['attributes']['role'] == 'controller']
+    if not controller:
+        return Result(result=ERROR, msg="No controller nodes found. Is the cluster healthy?", doc_url=doc_url)
+
+    print('')
+    checked_apics = {}
+    has_error = False
+    nodes_file_count_result = []
+    nodes_fatal_errors_result = []
+
+    for apic in controller:
+        attr = apic['fabricNode']['attributes']
+        if attr['address'] in checked_apics:
+            continue
+        checked_apics[attr['address']] = 1
+        node_id = attr['id']
+        node_name = attr['name']
+        node_title = 'Checking %s...' % node_name
+
+        try:
+            c = Connection(attr['address'])
+            c.username = username
+            c.password = password
+            c.log = LOG_FILE
+            c.connect()
+        except Exception as e:
+            data.append([node_id, '-', '-', 'ERROR: %s' % str(e)])
+            has_error = True
+            continue
+
+        try:
+            # Check if /firmware/tmp directory exists and count files
+            c.cmd('[ -d /firmware/tmp ] && ls -1 /firmware/tmp 2>/dev/null | wc -l || echo 0')
+            file_count = 0
+            for line in c.output.strip().split('\n'):
+                line = line.strip()
+                if line.isdigit():
+                    file_count = int(line)
+                    break
+
+            # Check for fatal errors in bootx logs
+            c.cmd('[ -d /var/log/bootx/logs ] && grep -Ri "fatal" /var/log/bootx/logs/* 2>/dev/null | wc -l || echo 0')
+            fatal_count = 0
+            for line in c.output.strip().split('\n'):
+                line = line.strip()
+                if line.isdigit():
+                    fatal_count = int(line)
+                    break
+
+            # Determine status
+            if file_count >= 1000:
+                status = 'FAIL - High file count'
+                data.append([node_id, str(file_count), str(fatal_count), status])
+                result = FAIL_UF
+                nodes_file_count_result.append(result)
+            elif fatal_count > 0:
+                status = 'WARNING - Fatal errors found'
+                data.append([node_id, str(file_count), str(fatal_count), status])
+                if result == PASS:
+                    result = MANUAL
+                nodes_fatal_errors_result.append(result)
+
+        except Exception as e:
+            data.append([node_id, '-', '-', 'ERROR: %s' % str(e)])
+            has_error = True
+            continue
+
+    if FAIL_UF in nodes_file_count_result:
+        result = FAIL_UF
+    if MANUAL in nodes_fatal_errors_result:
+        result = MANUAL
+    if has_error and result == PASS:
+        result = ERROR
+
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
+
 # ---- Script Execution ----
 
 
@@ -6049,6 +6150,7 @@ class CheckManager:
         post_upgrade_cb_check,
         validate_32_64_bit_image_check,
         fabric_link_redundancy_check,
+        bootx_firmware_tmp_check,
 
         # Faults
         apic_disk_space_faults_check,

docs/docs/validations.md

@@ -191,6 +191,7 @@ Items                                           | Defect       | This Script
 [Stale pconsRA Object][d26]                     | CSCwp22212   | :warning:{title="Deprecated"} | :no_entry_sign:
 [ISIS DTEPs Byte Size][d27]                     | CSCwp15375   | :white_check_mark: | :no_entry_sign:
 [Policydist configpushShardCont Crash][d28]     | CSCwp95515   | :white_check_mark: | 
+[Bootx Service failure log & firmware/tmp directory checks][d29]  | CSCwn37676  | :white_check_mark: | :no_entry_sign:
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -220,6 +221,7 @@ Items                                           | Defect       | This Script
 [d26]: #stale-pconsra-object
 [d27]: #isis-dteps-byte-size
 [d28]: #policydist-configpushshardcont-crash
+[d29]: #bootx_service_failure_log_and_firmware_tmp_directory_checks
 
 
 ## General Check Details
@@ -2604,6 +2606,21 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf
 If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade.
 
 
+### Bootx Service failure log & firmware/tmp directory checks
+
+Due to [CSCwn37676][62], ACI runs on releases 6.0(2) through 6.0(8) or 6.1(1) through 6.1(2) , upgrading to any target version with a high number of files in the `/firmware/tmp/` directory (1000 or more) or the presence of fatal errors in `/var/log/bootx/logs/` can cause the bootx service to fail, resulting in upgrade failures.
+
+The script performs two validations on each APIC:
+
+1. Checks if `/firmware/tmp/` directory contains 1000 or more files
+2. Searches for "fatal" errors in `/var/log/bootx/logs/`
+
+!!! warning
+    If this check fails, verify the bootx service status on the affected APIC(s) by running `systemctl status bootx`. If the service is not running, the APIC is already experiencing the issue and must be resolved before proceeding with the upgrade.
+
+!!! tip
+    Certain high churn logging configurations have been found to cause excessive files in `/firmware/tmp/while on non-fixed versions. If this check identifies issues, work with Cisco TAC to clean up excess files and resolve any bootx service failures before attempting the upgrade.
+
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
 [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html
@@ -2666,3 +2683,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC
 [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515
 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter
 [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression
+[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwn37676

tests/checks/bootx_firmware_tmp_check/fabricNode.json

@@ -0,0 +1,50 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.1",
+        "dn": "topology/pod-1/node-1",
+        "fabricSt": "commissioned",
+        "id": "1",
+        "model": "APIC-SERVER-L2",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "apic1",
+        "nodeType": "unspecified",
+        "podId": "1",
+        "role": "controller"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.2",
+        "dn": "topology/pod-1/node-2",
+        "fabricSt": "commissioned",
+        "id": "2",
+        "model": "APIC-SERVER-L2",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "apic2",
+        "nodeType": "unspecified",
+        "podId": "1",
+        "role": "controller"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.3",
+        "dn": "topology/pod-1/node-3",
+        "fabricSt": "commissioned",
+        "id": "3",
+        "model": "APIC-SERVER-L2",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "apic3",
+        "nodeType": "unspecified",
+        "podId": "1",
+        "role": "controller"
+      }
+    }
+  }
+]