Merge pull request #144 from sw360/feat/purl-qualifier-search

PURL qualifier-based search

Author: tngraf

ChangeLog.md

@@ -5,6 +5,12 @@
 
 # CaPyCli - Clearing Automation Python Command Line Tool for SW360
 
+## NEXT
+
+* `bom map`: The options `--dbx` and `-all` were replaced by `--matchmode`.
+* `bom map`: new `--matchmode` options `full-search` (report all best matches) and
+  `qualifier-match` (consider PackageURL qualifiers). See `Readme_Mapping.md`.
+
 ## 2.9.1
 
 * `bom map` will provide the `purl` from SW360 in the output BOM's components

Readme_Mapping.md

@@ -23,32 +23,58 @@ informs about the mapping result:
 
 * **`INVALID` (0)** => Invalid SBOM entry, could not get processed
 * **`FULL_MATCH_BY_ID` (1)** => Full match by identifier
-* **`FULL_MATCH_BY_HASH` (2)** => Full match by source file hash
+* **`FULL_MATCH_BY_HASH` (2)** => Full match by source or binary file hash
 * **`FULL_MATCH_BY_NAME_AND_VERSION` (3)** => Full match by name and version
 * **`MATCH_BY_FILENAME` (4)** => Match by source code filename
 * **`GOOD_MATCH_FOUND`** == `MATCH_BY_FILENAME` => successfully found a sufficiently good match
 * **`MATCH_BY_NAME` (5)** => Component found, but no version match
 * **`SIMILAR_COMPONENT_FOUND` (6)** => Component with similar name found, no version check done
 * **`NO_MATCH` (100)** => Component was not found
 
-In general you can say that the lower the number, the better the match.
+We consider lower numbers as better matches. By default, CaPyCli will stop the
+search when a "good" match (match code between 1 and 4) is found and add this
+release to the output BOM. If there are multiple good matches in SW360, the
+output thus depends on the order the results are returned by SW360 (or found in
+the CaPyCli cache).
+
+The "bom map --matchmode full-search" option allows to change that behaviour so that
+CaPyCli will always search through all releases in the API answer or cache, and
+report *all best* matches found. If there are matches by ID, other matches are
+ignored; matches by (source or binary) file hash will win over matches by name
+and version etc.
 
 ## Notes on id mapping / PackageURL mapping
 
-CaPyCli supports mapping releases by the PackageURL. As encoding of a
-PackageURL is not unique (some characters *may* use URL encoding, qualifiers
+CaPyCli supports mapping **releases** by the PackageURL. As encoding of a
+PackageURL is not unique (some characters may be percent-encoded, qualifiers
 can be given in random order etc.), we can't just do a string comparison, but
 instead *all* SW360 releases with PackageURLs (using external id `package-url`)
 are retrieved and decoded. When your input BOM specifies a `purl` field, then
 the PackageURL is compared field by field (type, namespace, name, version) for
 a `FULL_MATCH_BY_ID`.
 
-Also, components will be mapped by PackageURL and if a match is found, the
+Also, **components** will be mapped by PackageURL and if a match is found, the
 `capycli:componentId` property will be added to the output BOM item. Components
 can be identified directly by their external id `package-url` or as fallback
 also by the `package-url`s of their releases.
 
-PackageURL subpath and qualifiers are currently ignored during PURL matching.
+PackageURL **qualifiers** (like `?distro=alpine-3.21&package-id=3a23`) will be
+considered when using `bom map --matchmode qualifier-match`. In some cases,
+qualifiers are essential for correct mapping, but many scanners also include
+non-essential qualifiers in their SBOMs. And the distinction might be
+challenging: while `distro` is crucial for correct mapping of Alpine packages
+(same package release can have different patches in different Alpine releases),
+but for Debian, `distro` is unnecessary since package versions are already
+unique. So we use the following rules to balance accuracy and practicality:
+
+* Only the qualifiers specified in the input BOM are considered during matching,
+  qualifiers only present in SW360 releases are ignored. So you can control
+  matching by removing the unwanted qualifiers in your SBOM.
+* If one or more SW360 releases are found where *all* qualifiers specified in the
+  input BOM match, *only* these releases are added to the output BOM. Otherwise,
+  qualifiers will be ignored, so all release matches will be added.
+
+PackageURL subpath is currently ignored during PURL matching.
 
 ## Example 1: Very Simple, Full Match
 

capycli/bom/create_components.py

@@ -399,7 +399,7 @@ def update_release(self, cx_comp: Component, release_data: Dict[str, Any]) -> No
                         bom_purl = packageurl.PackageURL.from_string(
                             data["externalIds"][repository_type])
                         sw360_purls = PurlUtils.get_purl_list_from_sw360_object(release_data)
-                        id_match = PurlUtils.contains(sw360_purls, bom_purl)
+                        id_match = PurlUtils.contains(sw360_purls, bom_purl, compare_qualifiers=True)
                     except ValueError:
                         pass
                     if not id_match:

capycli/bom/map_bom.py

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: MIT
 # -------------------------------------------------------------------------------
 
+import copy
 import json
 import logging
 import os
@@ -58,6 +59,8 @@ def __init__(self) -> None:
         self.mode = MapMode.ALL
         self.purl_service: Optional[PurlService] = None
         self.no_match_by_name_only = True
+        self.full_search = False
+        self.qualifier_match = False
 
     def is_id_match(self, release: Dict[str, Any], component: Component) -> bool:
         """Determines whether this release is a match via identifier for the specified SBOM item"""
@@ -194,39 +197,51 @@ def map_bom_item(self, component: Component, check_similar: bool, result_require
             # first check: unique id
             if release["Sw360Id"] in result_release_ids or self.is_id_match(release, component):
                 self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_ID)
-                break
-
-            # second check: name AND version
-            if (component.name and release.get("Name")):
-                if release["ComponentId"] in result_component_ids:
-                    name_match = True
+                if self.full_search:
+                    continue
                 else:
-                    name_match = component.name.lower() == release["Name"].lower()
-                version_exists = "Version" in release
-                if (name_match
-                    and version_exists and component.version
-                        and (component.version.lower() == release["Version"].lower())):
-                    self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_NAME_AND_VERSION)
                     break
-            else:
-                name_match = False
 
-            # third check unique(?) file hashes
+            # second check unique(?) file hashes
             cmp_hash = CycloneDxSupport.get_source_file_hash(component)
             if (("SourceFileHash" in release)
                     and cmp_hash
                     and release["SourceFileHash"]):
                 if (cmp_hash.lower() == release["SourceFileHash"].lower()):
                     self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_HASH)
-                    break
+                    if self.full_search:
+                        continue
+                    else:
+                        break
 
             cmp_hash = CycloneDxSupport.get_binary_file_hash(component)
             if (("BinaryFileHash" in release)
                 and cmp_hash
                     and release["BinaryFileHash"]):
                 if (cmp_hash.lower() == release["BinaryFileHash"].lower()):
                     self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_HASH)
-                    break
+                    if self.full_search:
+                        continue
+                    else:
+                        break
+
+            # third check: name AND version
+            if (component.name and release.get("Name")):
+                if release["ComponentId"] in result_component_ids:
+                    name_match = True
+                else:
+                    name_match = component.name.lower() == release["Name"].lower()
+                version_exists = "Version" in release
+                if (name_match
+                    and version_exists and component.version
+                        and (component.version.lower() == release["Version"].lower())):
+                    self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_NAME_AND_VERSION)
+                    if self.full_search:
+                        continue
+                    else:
+                        break
+            else:
+                name_match = False
 
             # fourth check: source filename
             cmp_src_file = CycloneDxSupport.get_ext_ref_source_file(component)
@@ -235,7 +250,10 @@ def map_bom_item(self, component: Component, check_similar: bool, result_require
                     and release["SourceFile"]):
                 if cmp_src_file.lower() == release["SourceFile"].lower():
                     self.add_match_if_better(result, release, MapResult.MATCH_BY_FILENAME)
-                    break
+                    if self.full_search:
+                        continue
+                    else:
+                        break
 
             # fifth check: name and ANY version
             if name_match:
@@ -299,8 +317,10 @@ def get_release_details(href: str) -> Optional[Dict[str, Any]]:
                 release = get_release_details(href)
                 if release:
                     self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_ID)
-                # If we have release matches by PURL, we're done
-                return result
+                    if not self.full_search:
+                        return result
+            # If we have release matches by PURL, we're done
+            return result
 
         if result.component_hrefs:
             components += result.component_hrefs
@@ -343,29 +363,38 @@ def get_release_details(href: str) -> Optional[Dict[str, Any]]:
                     self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_ID)
                     break
 
-                # second check: name AND version (we don't need to check the name
-                # again as we checked it when compiling component list)
-                version_exists = "Version" in release
-                if (version_exists
-                        and ((component.version or "").lower() == release.get("Version", "").lower())):
-                    self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_NAME_AND_VERSION)
-                    break
-
-                # third check unique(?) file hashes
+                # second check unique(?) file hashes
                 cmp_hash = CycloneDxSupport.get_source_file_hash(component)
                 if (("SourceFileHash" in release)
                         and cmp_hash
                         and release["SourceFileHash"]):
                     if (cmp_hash.lower() == release["SourceFileHash"].lower()):
                         self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_HASH)
-                        break
+                        if self.full_search:
+                            continue
+                        else:
+                            break
 
                 cmp_hash = CycloneDxSupport.get_binary_file_hash(component)
                 if (("BinaryFileHash" in release)
                     and cmp_hash
                         and release["BinaryFileHash"]):
                     if (cmp_hash.lower() == release["BinaryFileHash"].lower()):
                         self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_HASH)
+                        if self.full_search:
+                            continue
+                        else:
+                            break
+
+                # third check: name AND version (we don't need to check the name
+                # again as we checked it when compiling component list)
+                version_exists = "Version" in release
+                if (version_exists
+                        and ((component.version or "").lower() == release.get("Version", "").lower())):
+                    self.add_match_if_better(result, release, MapResult.FULL_MATCH_BY_NAME_AND_VERSION)
+                    if self.full_search:
+                        continue
+                    else:
                         break
 
                 # fifth check: name and ANY version
@@ -506,6 +535,9 @@ def update_bom_item(self, component: Optional[Component], match: Dict[str, Any])
                     name=match.get("Name", ""),
                     version=match.get("Version", ""))
         else:
+            # copy component so we don't overwrite the input component
+            component = copy.deepcopy(component)
+
             # always overwrite the following properties
             name = match.get("Name", "")
             if name:
@@ -730,7 +762,9 @@ def map_bom_commons(self, component: Component) -> MapResult:
         # search release and component by purl which is independent of the component cache.
         if component.purl:
             result.component_hrefs = self.external_id_svc.search_components_by_purl(component.purl)
-            result.release_hrefs = self.external_id_svc.search_releases_by_purl(component.purl)
+            r = self.external_id_svc.search_releases_by_purl(component.purl, self.qualifier_match)
+            result.release_hrefs = r["hrefs"]
+            result.release_hrefs_results = r["results"]
 
         return result
 
@@ -815,9 +849,14 @@ def show_help(self) -> None:
         print("                          all = default, write everything to resulting SBOM")
         print("                          found = resulting SBOM shows only components that were found")
         print("                          notfound = resulting SBOM shows only components that were not found")
-        print("    --dbx                 relaxed Debian version handling: *completely* ignore Debian revision,")
-        print("                          so SBOM version 3.1 will match SW360 version 3.1-3.debian")
-        print("    -all                  also report matches for name, but different version")
+        print("    --matchmode MATCHMODE matching mode, comma separated list of:")
+        print("                          full-search = report best matches, don't abort on first match (recommended)")
+        print("                          all-versions = also report matches for name, but different version")
+        print("                          qualifier-match = consider qualifiers for PURL matching")
+        print("                          ignore-debian = ignore Debian revision in version comparison, so SBOM")
+        print("                                          version 3.1 will match SW360 version 3.1-3.debian")
+        print("    -all                  deprecated, please use --matchmode all-versions")
+        print("    --dbx                 deprecated, please use --matchmode ignore-debian")
 
     def run(self, args: Any) -> None:
         """Main method()"""
@@ -849,16 +888,29 @@ def run(self, args: Any) -> None:
         if args.verbose:
             self.verbosity = 2
 
-        if args.dbx:
+        if not args.matchmode:
+            args.matchmode = ""
+
+        if "ignore-debian" in args.matchmode or args.dbx:
+            if args.dbx:
+                print_yellow("bom map --dbx is deprecated, use --matchmode ignore-debian instead")
             print_text("Using relaxed debian version checks")
             self.relaxed_debian_parsing = True
 
         if args.mode:
             self.mode = args.mode
 
-        if args.all:
+        if "all-versions" in args.matchmode or args.all:
+            if args.all:
+                print_yellow("bom map -all is deprecated, use --matchmode all-versions instead")
             self.no_match_by_name_only = False
 
+        if "full-search" in args.matchmode:
+            self.full_search = True
+
+        if "qualifier-match" in args.matchmode:
+            self.qualifier_match = True
+
         print_text("Loading SBOM file", args.inputfile)
         try:
             sbom = CaPyCliBom.read_sbom(args.inputfile)

capycli/common/capycli_bom_support.py

@@ -60,6 +60,7 @@ class CycloneDxSupport():
     CDX_PROP_COMPONENT_ID = "capycli:componentId"
     CDX_PROP_FILENAME = "siemens:filename"
     CDX_PROP_MAPRESULT = "capycli:mapResult"
+    CDX_PROP_MAPRESULT_BY_ID = "capycli:mapResultById"
     CDX_PROP_SW360_HREF = "capycli:sw360Href"
     CDX_PROP_SW360_URL = "capycli:sw360Url"
     CDX_PROP_REL_STATE = "capycli:releaseMainlineState"

capycli/common/map_result.py

@@ -7,12 +7,20 @@
 # -------------------------------------------------------------------------------
 
 from typing import Any, List, Optional
+from enum import Enum
 
 from cyclonedx.model.component import Component
 
 from capycli.common.capycli_bom_support import CycloneDxSupport
 
 
+class MapResultByIdQualifiers(Enum):
+    FULL_MATCH = "qualifiers-full-match"
+    IGNORED = "qualifiers-ignored"
+    UNKNOWN = "qualifiers-unknown-match"
+    NO_QUALIFIER_MAPPING = ""
+
+
 class MapResult:
     """Result of mapping a SBOM item to the list of releases"""
 
@@ -50,8 +58,22 @@ def __init__(self, component: Optional[Component] = None) -> None:
         self.result: str = MapResult.NO_MATCH
         self._component_hrefs: List[str] = []
         self._release_hrefs: List[str] = []
+        self._release_hrefs_results: List[str] = []
         self.releases: List[Any] = []
 
+    @property
+    def release_hrefs_results(self) -> list[str]:
+        return self._release_hrefs_results
+
+    @release_hrefs_results.setter
+    def release_hrefs_results(self, value: list[str]) -> None:
+        self._release_hrefs_results = value
+        if not self.input_component or not value:
+            return
+        CycloneDxSupport.update_or_set_property(
+            self.input_component, CycloneDxSupport.CDX_PROP_MAPRESULT_BY_ID,
+            " ".join(value))
+
     @property
     def component_hrefs(self) -> List[str]:
         return self._component_hrefs