Test multiple markers, arg validation, fix command print

Author: avi-jois

README.rst

@@ -60,7 +60,8 @@ Windows and ``pythonX.Y -m piptools compile`` on other systems.
 ``pip-compile`` should be run from the same virtual environment as your
 project so conditional dependencies that require a specific Python version,
 or other environment markers, resolve relative to your project's
-environment.
+environment. If you need to resolve dependencies for a different environment,
+see `Cross-environment`_ for some solutions.
 
 **Note**: If ``pip-compile`` finds an existing ``requirements.txt`` file that
 fulfils the dependencies then no changes will be made, even if updates are
@@ -529,6 +530,8 @@ We suggest to use the ``{env}-requirements.txt`` format
 (ex: ``win32-py3.7-requirements.txt``, ``macos-py3.10-requirements.txt``, etc.).
 
 
+.. _Cross-environment:
+
 Cross-environment usage of ``requirements.in``/``requirements.txt`` and ``pip-compile``
 =======================================================================================
 
@@ -559,6 +562,22 @@ when targetting a different environment so the environment is fully defined.
 
 .. _PEP 508 environment markers: https://www.python.org/dev/peps/pep-0508/#environment-markers
 
+For example, if you wanted to evaluate ``requirements.in`` for a typical Linux machine:
+
+.. code-block:: bash
+
+    $ pip-compile requirements.in \
+        --override-environment os_name posix \
+        --override-environment sys_platform linux \
+        --override-environment platform_machine x86_64 \
+        --override-environment platform_python_implementation CPython \
+        --override-environment platform_release '' \
+        --override-environment platform_version '' \
+        --override-environment python_version 3.11 \
+        --override-environment python_full_version 3.11.0 \
+        --override-environment implementation_name cpython \
+        --override-environment implementation_version 3.11.0
+
 Other useful tools
 ==================
 

piptools/scripts/compile.py

@@ -8,14 +8,15 @@
 from typing import IO, Any, BinaryIO, cast
 
 import click
-from build import BuildBackendException
 from build.util import project_wheel_metadata
 from click.utils import LazyFile, safecall
 from pip._internal.commands import create_command
 from pip._internal.req import InstallRequirement
 from pip._internal.req.constructors import install_req_from_line
 from pip._internal.utils.misc import redact_auth_from_url
 
+from build import BuildBackendException
+
 from .._compat import parse_requirements
 from ..cache import DependencyCache
 from ..exceptions import NoCandidateFound, PipToolsError
@@ -25,12 +26,14 @@
 from ..repositories.base import BaseRepository
 from ..resolver import BacktrackingResolver, LegacyResolver
 from ..utils import (
+    PEP508_ENVIRONMENT_MARKERS,
     UNSAFE_PACKAGES,
     dedup,
     drop_extras,
     is_pinned_requirement,
     key_from_ireq,
     parse_requirements_from_wheel_metadata,
+    validate_environment_overrides,
 )
 from ..writer import OutputWriter
 
@@ -308,6 +311,7 @@ def _determine_linesep(
     type=(str, str),
     help="Specify an environment marker to override."
     "This can be used to fetch requirements for a different platform",
+    callback=validate_environment_overrides,
 )
 def cli(
     ctx: click.Context,
@@ -347,7 +351,7 @@ def cli(
     emit_index_url: bool,
     emit_options: bool,
     unsafe_package: tuple[str, ...],
-    override_environment: tuple[tuple[str, str], ...],
+    override_environment: dict[str, str],
 ) -> None:
     """
     Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg,
@@ -447,20 +451,7 @@ def cli(
 
         def overriden_environment() -> dict[str, str]:
             return {
-                k: env_dict.get(k, default_env[k])
-                for k in [
-                    "implementation_name",
-                    "implementation_version",
-                    "os_name",
-                    "platform_machine",
-                    "platform_release",
-                    "platform_system",
-                    "platform_version",
-                    "python_full_version",
-                    "platform_python_implementation",
-                    "python_version",
-                    "sys_platform",
-                ]
+                k: env_dict.get(k, default_env[k]) for k in PEP508_ENVIRONMENT_MARKERS
             }
 
         pip._vendor.packaging.markers.default_environment = overriden_environment

piptools/utils.py

@@ -47,6 +47,25 @@
     "--no-reuse-hashes",
 }
 
+# Only certain environment markers are allowed in requirement specifications.
+# Validate that overrides use a valid marker in order to provide better debug
+# feedback to the user.
+PEP508_ENVIRONMENT_MARKERS = [
+    "os_name",
+    "sys_platform",
+    "platform_machine",
+    "platform_python_implementation",
+    "platform_release",
+    "platform_system",
+    "platform_version",
+    "python_version",
+    "python_full_version",
+    "implementation_name",
+    "implementation_version",
+    # Note that 'extra' is omitted here because that should be set at the wheel
+    # level, not the runtime level.
+]
+
 
 def key_from_ireq(ireq: InstallRequirement) -> str:
     """Get a standardized key for an InstallRequirement."""
@@ -389,13 +408,18 @@ def get_compile_command(click_ctx: click.Context) -> str:
             else:
                 if isinstance(val, str) and is_url(val):
                     val = redact_auth_from_url(val)
+
                 if option.name == "pip_args_str":
                     # shlex.quote() would produce functional but noisily quoted results,
                     # e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"''
                     # Instead, we try to get more legible quoting via repr:
-                    left_args.append(f"{option_long_name}={repr(val)}")
+                    quoted_val = repr(val)
+                elif isinstance(val, (tuple, list)):
+                    quoted_val = " ".join([shlex.quote(str(v)) for v in val])
                 else:
-                    left_args.append(f"{option_long_name}={shlex.quote(str(val))}")
+                    quoted_val = shlex.quote(str(val))
+
+                left_args.append(f"{option_long_name}={quoted_val}")
 
     return " ".join(["pip-compile", *sorted(left_args), *sorted(right_args)])
 
@@ -522,3 +546,16 @@ def parse_requirements_from_wheel_metadata(
             markers=parts.markers,
             extras=parts.extras,
         )
+
+
+def validate_environment_overrides(
+    _ctx: click.Context,
+    _param: str,
+    value: list[tuple[str, str]],
+) -> list[tuple[str, str]]:
+    for key, _ in value:
+        if key not in PEP508_ENVIRONMENT_MARKERS:
+            raise click.BadParameter(
+                f"Override key '{key}' must be one of " f"{PEP508_ENVIRONMENT_MARKERS}!"
+            )
+    return value

tests/test_cli_compile.py

@@ -2973,7 +2973,7 @@ def test_cross_fetch_top_level(fake_dists, runner, platform):
             """
         )
 
-    assert out.exit_code == 0, out
+    assert out.exit_code == 0, out.stderr
     assert expected_output == out.stdout
 
 
@@ -3041,5 +3041,136 @@ def test_cross_fetch_transitive_deps(
             + expected_output
         )
 
-    assert out.exit_code == 0, out
+    assert out.exit_code == 0, out.stderr
+    assert expected_output == out.stdout
+
+
+def test_multiple_env_overrides(fake_dists, runner):
+    """
+    test passing multiple `--override-environment` evaluates top level
+    requirements correctly.
+    """
+
+    # Use arbitrary values for the markers that aren't reasonable values that
+    # might come from the test environment.
+    with open("requirements.in", "w") as req_in:
+        req_in.write('small-fake-a==0.1 ; sys_platform == "foo"\n')
+        req_in.write('small-fake-b==0.2 ; sys_platform == "bar"\n')
+        req_in.write('small-fake-c==0.3 ; implementation_name == "baz"\n')
+
+    out = runner.invoke(
+        cli,
+        [
+            "--output-file",
+            "-",
+            "--quiet",
+            "--find-links",
+            fake_dists,
+            "--no-annotate",
+            "--no-emit-options",
+            "--no-header",
+            "--override-environment",
+            "sys_platform",
+            "foo",
+            "--override-environment",
+            "implementation_name",
+            "baz",
+        ],
+    )
+
+    expected_output = dedent(
+        """\
+        small-fake-a==0.1 ; sys_platform == "foo"
+        small-fake-c==0.3 ; implementation_name == "baz"
+        """
+    )
+
+    assert out.exit_code == 0, out.stderr
     assert expected_output == out.stdout
+
+
+@pytest.mark.parametrize(
+    "marker",
+    (
+        "os_name",
+        "sys_platform",
+        "platform_machine",
+        "platform_python_implementation",
+        "platform_release",
+        "platform_version",
+        "python_version",
+        "python_full_version",
+        "implementation_name",
+        "implementation_version",
+    ),
+)
+def test_all_env_overrides(fake_dists, runner, marker):
+    """
+    test that each valid `--override-environment` key can be used to select
+    requirements.
+    """
+
+    # Use arbitrary values for the markers that aren't reasonable values that
+    # might come from the test environment.
+    with open("requirements.in", "w") as req_in:
+        req_in.write(f'small-fake-a==0.1 ; {marker} == "foo"\n')
+        req_in.write(f'small-fake-b==0.2 ; {marker} == "bar"\n')
+
+    for marker_value in ["foo", "bar"]:
+        out = runner.invoke(
+            cli,
+            [
+                "--output-file",
+                "-",
+                "--quiet",
+                "--find-links",
+                fake_dists,
+                "--no-annotate",
+                "--no-emit-options",
+                "--no-header",
+                "--override-environment",
+                marker,
+                marker_value,
+            ],
+        )
+
+        if marker_value == "foo":
+            expected_output = dedent(
+                f"""\
+                small-fake-a==0.1 ; {marker} == "foo"
+                """
+            )
+        else:
+            expected_output = dedent(
+                f"""\
+                small-fake-b==0.2 ; {marker} == "bar"
+                """
+            )
+
+        assert out.exit_code == 0, out.stderr
+        assert expected_output == out.stdout
+
+
+def test_invalid_env_override(runner):
+    """
+    test passing an invalid `--override-environment` key triggers an error.
+    """
+
+    with open("requirements.in", "w"):
+        pass
+
+    out = runner.invoke(
+        cli,
+        [
+            "--output-file",
+            "-",
+            "--override-environment",
+            "foo",
+            "bar",
+        ],
+    )
+
+    expected_error = "Invalid value for '--override-environment'"
+
+    assert out.exit_code == 2
+    assert expected_error in out.stderr

tests/test_utils.py

@@ -362,6 +362,20 @@ def test_is_url_requirement_filename(caplog, from_line, line):
             ["--pip-args", "--disable-pip-version-check --isolated"],
             "pip-compile --pip-args='--disable-pip-version-check --isolated'",
         ),
+        (
+            ["--override-environment", "os_name", "posix"],
+            "pip-compile --override-environment=os_name posix",
+        ),
+        # Check that an override value with spaces and an empty override are
+        # properly escaped.
+        (
+            ["--override-environment", "platform_version", "multiple words"],
+            "pip-compile --override-environment=platform_version 'multiple words'",
+        ),
+        (
+            ["--override-environment", "platform_release", ""],
+            "pip-compile --override-environment=platform_release ''",
+        ),
         pytest.param(
             ["--extra-index-url", "https://username:password@example.com/"],
             "pip-compile --extra-index-url='https://username:****@example.com/'",