Merge pull request #172 from sw360/171-support-python-dependency-detection-for-python-projects-using-poetry-2x

adapt `getdependencies python` to the Poetry 2.x pyproject.toml…

Author: tngraf

ChangeLog.md

@@ -20,6 +20,7 @@
 * Dependency updates.
 * `project prerequisites` now has a summary at the end of the output to show how many
   components have been scanned and how many warnings and errors there are.
+* Adapt `getdependencies python` to the Poetry 2.x pyproject.toml format.
 
 ## 2.9.1
 

capycli/dependencies/python.py

@@ -68,6 +68,7 @@ class GetPythonDependencies(capycli.common.script_base.ScriptBase):
 
     def __init__(self) -> None:
         self.verbose = False
+        self.proj_file_override = ""
 
     @staticmethod
     def normalize_packagename(name: str) -> str:
@@ -404,8 +405,7 @@ def determine_file_type(self, full_filename: str) -> InputFileType:
             LOG.debug("Guessing requirements file")
             return InputFileType.REQUIREMENTS
 
-        if (filename == "poetry.lock") or (filename == "poetry2.lock"):
-            # "poetry2.lock is only for unit test
+        if (filename == "poetry.lock"):
             data = self.read_poetry_lock_file(full_filename)
             if data:
                 LOG.debug("Guessing poetry.lock file")
@@ -510,6 +510,13 @@ def add_entry(self,
             else:
                 LOG.warning(f"Dependency {dep2} not found!")
 
+    @staticmethod
+    def get_pure_dep_name(name: str) -> str:
+        """Get pure dependency name (without extras)."""
+        if "(" in name:
+            return name.split("(")[0].strip()
+        return name
+
     def get_lock_file_entries_for_sbom(self,
                                        pyproject_file: str,
                                        all_entries: List[LockFileEntry]) -> List[LockFileEntry]:
@@ -521,10 +528,17 @@ def get_lock_file_entries_for_sbom(self,
             # => return all dependencies
             return all_entries
 
-        cfg = pyproject_info["tool"]["poetry"]
+        poetry2xflag = False
+        if "project" in pyproject_info:
+            cfg = pyproject_info["project"]
+            poetry2xflag = True
+        else:
+            cfg = pyproject_info["tool"]["poetry"]
         # get only real dependencies
         dependencies = cfg.get("dependencies", [])
         for dep in dependencies:
+            if poetry2xflag:
+                dep = self.get_pure_dep_name(dep)
             dep_name = GetPythonDependencies.normalize_packagename(dep)
             if dep_name.lower() == "python":
                 # ignore python (version) 'dependency'
@@ -536,7 +550,7 @@ def get_lock_file_entries_for_sbom(self,
                 LOG.warning(f"Dependency {dep_name} not found!")
 
         # are there other groups of dependencies?
-        dep_groups = cfg.get("group")
+        dep_groups = pyproject_info["tool"]["poetry"].get("group")
         for group in dep_groups:
             print_yellow("Ignoring dependency group " + group)
             for dep in dep_groups[group].get("dependencies", []):
@@ -546,7 +560,12 @@ def get_lock_file_entries_for_sbom(self,
 
     def sbom_from_poetry_lock_file(self, filename: str, search_meta_data: bool, package_source: str = "") -> Bom:
         folder = os.path.dirname(filename)
-        pyproject_file = os.path.join(folder, "pyproject.toml")
+
+        if self.proj_file_override:
+            # override for unit tests
+            pyproject_file = self.proj_file_override
+        else:
+            pyproject_file = os.path.join(folder, "pyproject.toml")
         creator = SbomCreator()
         sbom = creator.create([], addlicense=True, addprofile=True, addtools=True)
         entry_list_all = self.get_all_lock_file_entries(filename)

tests/fixtures/poetry2.lock tests/fixtures/poetry_1.8.3/poetry.locktests/fixtures/poetry2.lock renamed to tests/fixtures/poetry_1.8.3/poetry.lock

@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText: (c) 2018-2024 Siemens
+# SPDX-License-Identifier: MIT
+
+[tool.poetry]
+name = "capycli"
+version = "2.6.0.dev1"
+description = "CaPyCli - Clearing Automation Python Command Line Interface for SW360"
+authors = ["Thomas Graf <[email protected]>"]
+license = "MIT"
+readme="Readme.md"
+repository = "https://github.com/sw360/capycli"
+homepage = "https://github.com/sw360/capycli"
+keywords = ["sw360", "cli, automation", "license", "compliance", "clearing"]
+include = [
+    "LICENSE.md",
+    { path = "capycli/data/granularity_list.csv", format = "wheel" },
+    { path = "capycli/data/__init__.py", format = "wheel" },
+    { path = "capycli/data/granularity_list.csv", format = "sdist" },
+    { path = "capycli/data/__init__.py", format = "sdist" },
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3 :: Only",
+]
+
+
+[tool.poetry.urls]
+issues = "https://github.com/sw360/capycli/issues"
+
+[tool.poetry.scripts]
+capycli = "capycli.main.cli:main"
+
+[tool.poetry.dependencies]
+python = "^3.8" # drop support for 3.6 and 3.7 because of wheel and cli-support
+colorama = "^0.4.3"
+requests = "^2.31.0" # fix CVE-2023-32681 
+semver = "3.0.2"
+packageurl-python = "^0.15.6"
+pyjwt = "^1.7.1"
+openpyxl = "^3.0.3"
+requirements-parser = "0.11.0"
+sw360 = "^1.5.0"
+wheel = "^0.38.4"
+cli-support = "2.0.1"
+chardet = "5.2.0"
+cyclonedx-python-lib = "^8.0.0"
+tomli = "^2.0.2"
+dateparser = "^1.1.8"
+urllib3 = "*"
+importlib-resources = "^5.12.0"
+beautifulsoup4 = "^4.11.1"
+jsonschema = "^4.23.0"
+
+[tool.poetry.group.dev.dependencies]
+flake8 = ">=3.7.8"
+coverage = "^5.4"
+responses = "0.24.1"
+pytest = "7.4.3"
+vcrpy = "4.4.0"  # arrghh, no version of vcrpy supports urllib3 >= 2
+cli-test-helpers = "^3.1.0"
+isort = "^5.12.0"
+mypy = "^1.8.0"
+types-colorama = "^0.4.15.12"
+types-urllib3 = "^1.26.25.14"
+types-openpyxl = "^3.1.0.32"
+types-python-dateutil = "^2.8.19.14"
+types-requests = "2.31.0.6"  # this is the last version that uses urllib3 < 2
+types-beautifulsoup4 = "^4.12.0.20240106"
+codespell = "^2.2.6"
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"
+
+[tool.pytest.ini_options]
+filterwarnings = [
+    # note the use of single quote below to denote "raw" strings in TOML
+    "ignore:pkg_resources is deprecated as an API:DeprecationWarning",
+    "ignore:Both `id` and `name` have been supplied - `name` will be ignored!",
+    # cyclonedx-python-lib - UserWarning: The Component this BOM is describing None...
+    "ignore::UserWarning",
+    # cyclonedx-python-lib - DeprecationWarning: `@.tools` is deprecated from CycloneDX v1.5 onwards
+    "ignore::DeprecationWarning"
+]
+
+[tool.mypy]
+exclude = [
+    "/tests",
+]
+
+show_error_codes = true
+pretty = true
+
+warn_unreachable = true
+allow_redefinition = true
+
+### Strict mode ###
+warn_unused_configs         = true
+disallow_subclassing_any    = true
+
+disallow_any_generics       = true
+disallow_untyped_calls      = true
+disallow_untyped_defs       = true
+disallow_incomplete_defs    = true
+check_untyped_defs          = true
+disallow_untyped_decorators = true
+no_implicit_optional        = true
+warn_redundant_casts        = true
+warn_unused_ignores         = true
+no_implicit_reexport        = true
+
+[tool.codespell]
+skip = "./htmlcov/*,./_internal_tests_/*,./__internal__/*,./tests/fixtures/*,*.svg,./capycli/data/granularity_list.csv,./ComponentCache.*"
+ignore-words-list = "manuel, assertIn"