Merge branch 'feat/false_positives' into 'master'

feat: add a global list of excluded CVEs

Closes IDF-10301

See merge request espressif/esp-idf-sbom!42

Author: fhrbata

.pre-commit-config.yaml

@@ -44,3 +44,13 @@ repos:
     hooks:
       - id: check-copyright
         args: ['--config', 'check_copyright_config.yaml']
+
+  - repo: local
+    hooks:
+      - id: validate-excluded-cves
+        name: Validate Excluded CVEs
+        entry: test/validate_excluded_cves.py
+        language: python
+        files: 'excluded_cves.yaml'
+        additional_dependencies:
+          - PyYAML

esp_idf_sbom/libsbom/nvd.py

@@ -12,6 +12,8 @@
 import urllib.request
 from typing import Any, Dict, List
 
+import yaml
+
 from esp_idf_sbom.libsbom import log
 
 HINT = '''\
@@ -44,8 +46,11 @@ def nvd_request(params: str) -> List[Dict[str, Any]]:
         log.debug('NVD request headers:')
         log.debug('\n'.join([f'{h}: {v}' for h, v in req.header_items()]))
 
+        # NVD recommends waiting for six seconds between requests.
+        # https://nvd.nist.gov/developers/start-here
+        time.sleep(6)
         try:
-            with urllib.request.urlopen(req, timeout=30) as res:
+            with urllib.request.urlopen(req, timeout=60) as res:
                 data = json.loads(res.read().decode())
 
         except urllib.error.HTTPError as e:
@@ -68,9 +73,10 @@ def nvd_request(params: str) -> List[Dict[str, Any]]:
         except OSError as e:
             # We may encounter a read error from the underlying socket. If that happens,
             # allow up to 3 retries along with 503 HTTP error.
-            unavailable_cnt += 1
-            log.warn(f'Unable to read response from NVD server: {e}. Retrying({unavailable_cnt}).')
             if unavailable_cnt < 3:
+                unavailable_cnt += 1
+                log.warn(f'Unable to read response from NVD server: {e}. Retrying({unavailable_cnt}) in 10 second...')
+                time.sleep(10)
                 continue
             raise
 
@@ -86,6 +92,36 @@ def nvd_request(params: str) -> List[Dict[str, Any]]:
     return vulns
 
 
+def get_excluded_cves(cache: Dict[str, Dict[str, Any]]={}) -> Dict[str, Any]:
+    """Retrieve the YAML file from the esp-idf-sbom repository, which includes a list of excluded CVEs."""
+
+    if 'cves' in cache:
+        # Download the excluded CVEs once per script run, not for each check.
+        return cache['cves']
+
+    cves: Dict[str, Any] = {}
+    url = 'https://raw.githubusercontent.com/espressif/esp-idf-sbom/master/excluded_cves.yaml'
+    req = urllib.request.Request(url)
+
+    for retry in range(1, 4):
+        try:
+            with urllib.request.urlopen(req, timeout=30) as res:
+                cves = yaml.safe_load(res.read().decode())
+            break
+
+        except urllib.error.HTTPError as e:
+            log.warn(f'Cannot download list of excluded CVEs: {e}. Retrying({retry}) ...')
+
+        except yaml.YAMLError as e:
+            log.warn(f'Cannot load list of excluded CVEs: {e}')
+            break
+    else:
+        log.warn(f'Failed to download list of excluded CVEs')
+
+    cache['cves'] = cves
+    return cves
+
+
 # https://nvd.nist.gov/developers/vulnerabilities
 def check(cpe: str, search_name: bool=False) -> List[Dict[str, Any]]:
     """Checks given CPE against NVD and returns its reponse."""
@@ -96,13 +132,19 @@ def check(cpe: str, search_name: bool=False) -> List[Dict[str, Any]]:
     if not search_name:
         return cpe_vulns
 
+    # Obtain the list of excluded CVEs to filter them out from the unanalyzed CVEs provided by NVD.
+    excluded_cves = get_excluded_cves()
+
     # Check for vulnerabilities using the package name from CPE and do keywordSearch.
     pkg_name = cpe.split(':')[4]
     keyword_vulns = nvd_request(f'keywordSearch={pkg_name}')
 
     for vuln in keyword_vulns:
         if vuln['cve']['vulnStatus'] in ['Received', 'Awaiting Analysis', 'Undergoing Analysis']:
             # CVE not analyzed in NVD, include it in the results.
+            if vuln['cve']['id'] in excluded_cves:
+                # This CVE was previously analyzed and determined to be a false positive, unrelated to ESP-IDF.
+                continue
             cpe_vulns.append(vuln)
 
     return cpe_vulns

excluded_cves.yaml

@@ -0,0 +1,6 @@
+# This file lists excluded CVEs to be used with the --name option during
+# esp-idf-sbom check or manifest check, helping to exclude false positives for
+# unanalyzed CVEs in the NVD database.
+
+CVE-2021-47262: related to the Linux kernel
+CVE-2021-47264: related to the Linux kernel

test/validate_excluded_cves.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import re
+import sys
+
+import yaml
+
+CVE_RE = re.compile(r'CVE-\d{4}-\d{4,7}')
+
+for fn in sys.argv[1:]:
+    with open(fn, 'r') as f:
+        cves = yaml.safe_load(f)
+    for cve in cves:
+        match = CVE_RE.match(cve)
+        if not match:
+            sys.exit(f'{cve} does not match CVE format')