Fix pip-sync --python-executable evaluating markers for the wrong environment

Author: gschaffner

piptools/scripts/sync.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
 import itertools
+import json
 import os
 import shlex
 import shutil
 import sys
 from pathlib import Path
+from subprocess import run  # nosec
 from typing import cast
 
 import click
@@ -100,6 +102,21 @@ def cli(
 
     if python_executable:
         _validate_python_executable(python_executable)
+        environment = json.loads(
+            run(  # nosec
+                [
+                    python_executable,
+                    "-m",
+                    "pip",
+                    "inspect",
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+            ).stdout
+        )["environment"]
+    else:
+        environment = None
 
     install_command = cast(InstallCommand, create_command("install"))
     options, _ = install_command.parse_args([])
@@ -113,7 +130,9 @@ def cli(
     )
 
     try:
-        merged_requirements = sync.merge(requirements, ignore_conflicts=force)
+        merged_requirements = sync.merge(
+            requirements, ignore_conflicts=force, environment=environment
+        )
     except PipToolsError as e:
         log.error(str(e))
         sys.exit(2)
@@ -128,7 +147,9 @@ def cli(
         local_only=python_executable is None,
         paths=paths,
     )
-    to_install, to_uninstall = sync.diff(merged_requirements, installed_dists)
+    to_install, to_uninstall = sync.diff(
+        merged_requirements, installed_dists, environment
+    )
 
     install_flags = _compose_install_flags(
         finder,

piptools/sync.py

@@ -4,6 +4,7 @@
 import os
 import sys
 import tempfile
+from functools import wraps
 from subprocess import run  # nosec
 from typing import Deque, Iterable, Mapping, ValuesView
 
@@ -40,6 +41,41 @@
 ]
 
 
+def patch_match_markers() -> None:
+    """
+    Monkey patches pip._internal.req.InstallRequirement.match_markers to allow
+    us to pass environment other than "extra".
+    """
+
+    @wraps(InstallRequirement.match_markers)
+    def match_markers(
+        self: InstallRequirement,
+        extras_requested: Iterable[str] | None = None,
+        environment: dict[str, str] | None = None,
+    ) -> bool:
+        if environment is None:
+            environment = {}
+
+        assert "extra" not in environment
+
+        if not extras_requested:
+            # Provide an extra to safely evaluate the markers
+            # without matching any extra
+            extras_requested = ("",)
+        if self.markers is not None:
+            return any(
+                self.markers.evaluate({"extra": extra, **environment})
+                for extra in extras_requested
+            )
+        else:
+            return True
+
+    InstallRequirement.match_markers = match_markers
+
+
+patch_match_markers()
+
+
 def dependency_tree(
     installed_keys: Mapping[str, Distribution], root_key: str
 ) -> set[str]:
@@ -93,15 +129,17 @@ def get_dists_to_ignore(installed: Iterable[Distribution]) -> list[str]:
 
 
 def merge(
-    requirements: Iterable[InstallRequirement], ignore_conflicts: bool
+    requirements: Iterable[InstallRequirement],
+    ignore_conflicts: bool,
+    environment: dict[str, str] | None = None,
 ) -> ValuesView[InstallRequirement]:
     by_key: dict[str, InstallRequirement] = {}
 
     for ireq in requirements:
         # Limitation: URL requirements are merged by precise string match, so
         # "file:///example.zip#egg=example", "file:///example.zip", and
         # "example==1.0" will not merge with each other
-        if ireq.match_markers():
+        if ireq.match_markers(environment=environment):
             key = key_from_ireq(ireq)
 
             if not ignore_conflicts:
@@ -158,6 +196,7 @@ def diff_key_from_req(req: Distribution) -> str:
 def diff(
     compiled_requirements: Iterable[InstallRequirement],
     installed_dists: Iterable[Distribution],
+    environment: dict[str, str] | None = None,
 ) -> tuple[set[InstallRequirement], set[str]]:
     """
     Calculate which packages should be installed or uninstalled, given a set
@@ -172,13 +211,15 @@ def diff(
     pkgs_to_ignore = get_dists_to_ignore(installed_dists)
     for dist in installed_dists:
         key = diff_key_from_req(dist)
-        if key not in requirements_lut or not requirements_lut[key].match_markers():
+        if key not in requirements_lut or not requirements_lut[key].match_markers(
+            environment=environment
+        ):
             to_uninstall.add(key)
         elif requirements_lut[key].specifier.contains(dist.version):
             satisfied.add(key)
 
     for key, requirement in requirements_lut.items():
-        if key not in satisfied and requirement.match_markers():
+        if key not in satisfied and requirement.match_markers(environment=environment):
             to_install.add(requirement)
 
     # Make sure to not uninstall any packages that should be ignored