-
Notifications
You must be signed in to change notification settings - Fork 5
Expand file tree
/
Copy pathip_pinned_transport.py
More file actions
376 lines (318 loc) · 13.5 KB
/
Copy pathip_pinned_transport.py
File metadata and controls
376 lines (318 loc) · 13.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
"""IP-pinned httpx transports that close the DNS-rebinding TOCTOU.
The default signing fetchers (JWKS + revocation-list, sync + async)
resolve the target hostname via :func:`resolve_and_validate_host`,
then hand the URL back to httpx — which resolves the hostname a
second time at connect. A malicious origin with ``TTL=0`` can return
a safe IP on the first lookup (passing SSRF validation) and a
private IP or cloud-metadata address on the second.
This module closes that gap. :func:`build_ip_pinned_transport`
resolves once, picks the first IP that passes the SSRF validator,
and returns an :class:`httpx.HTTPTransport` wired to a custom
:mod:`httpcore` network backend that translates the pinned
hostname → IP at connect time. TLS certificate validation still
runs against the original hostname (httpcore passes it separately
as ``server_hostname`` during the TLS handshake), so cert CN/SAN
matching is unaffected.
The transport is single-host-scoped. Reusing it for a DIFFERENT
hostname would bypass the pin and either connect to the wrong IP
or fail SSRF re-resolution. Build one transport per hostname you
need to reach; the existing fetchers do this per-call.
Naming conventions
------------------
* Classes use the ``Async`` CapWords prefix
(:class:`AsyncIpPinnedTransport`).
* Factory functions that BUILD an async transport use
``build_async_*`` (:func:`build_async_ip_pinned_transport`). The
factory itself is synchronous — it returns an async transport.
* The legacy ``abuild_*`` alias remains for backward-compatibility
but is deprecated.
Dependency on httpcore internals
--------------------------------
We reach into httpcore at two points:
1. ``httpcore.ConnectionPool(network_backend=...)`` — public API.
2. ``httpcore._backends.sync.SyncBackend`` /
``httpcore._backends.anyio.AnyIOBackend`` — underscore-prefixed
path, nominally private. The backend classes are the documented
default-backend implementations, and the ``network_backend`` kwarg
is the sanctioned extension point, but the stability of the
backend class names themselves isn't guaranteed.
Mitigations:
* ``pyproject.toml`` pins ``httpcore>=1.0,<2.0``.
* :class:`adcp.signing.ip_pinned_transport` exports the backend
signatures from a contract test that fails on import if upstream
changes them — see
``tests/conformance/signing/test_ip_pinned_transport.py``.
"""
from __future__ import annotations
import ssl
import warnings
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import httpcore
import httpx
import idna
# Private but documented-as-the-default-backend implementations. The
# underscore prefix is a stability hazard; the contract test in
# tests/conformance/signing/test_ip_pinned_transport.py fails if the
# signatures we rely on change, so a silent upstream break becomes a
# CI failure instead of a latent security regression.
from httpcore._backends.anyio import AnyIOBackend as _AnyIOBackend
from httpcore._backends.sync import SyncBackend as _SyncBackend
from adcp.signing._idna_canonicalize import canonicalize_host
from adcp.signing.jwks import resolve_and_validate_host
if TYPE_CHECKING:
from httpcore._backends.base import SOCKET_OPTION
__all__ = [
"AsyncIpPinnedTransport",
"IpPinnedTransport",
"abuild_ip_pinned_transport", # deprecated alias; remove next release
"build_async_ip_pinned_transport",
"build_ip_pinned_transport",
]
def _build_ssl_context() -> ssl.SSLContext:
"""Standard cert-validating TLS context. ``check_hostname`` stays True
so the hostname-in-cert-SAN match runs against the URL's original
host (the hostname httpcore passes as ``server_hostname`` during
the handshake), not the pinned IP.
"""
return ssl.create_default_context()
def _normalize_pin_host(host: str) -> str:
"""Normalize a hostname for byte-equal comparison.
Delegates to :func:`canonicalize_host` — strips a single trailing
dot, ASCII-lowercases, short-circuits IP literals (v4 and v6,
bracketed or not) before IDNA, and otherwise encodes via
IDNA-2008 (UTS#46 with ``transitional_processing=False``).
Matches the JWKS fetcher's ``resolve_and_validate_host`` so a pin
set on ``straße.de`` collapses to the same A-label httpx will
pass to httpcore at connect time.
Falls back to the raw input on IDNA encode failure so the
comparison just fails cleanly instead of raising inside
connect_tcp.
"""
try:
return canonicalize_host(host)
except (idna.IDNAError, UnicodeError, UnicodeEncodeError):
return host.lower().rstrip(".")
class _IpPinnedSyncBackend(_SyncBackend):
"""httpcore sync backend that connects by IP for one pinned hostname.
Delegates to the parent's ``connect_tcp`` after swapping the
host argument from the hostname to the pre-resolved IP. All
other methods (``connect_unix_socket``) pass through unchanged.
**Fails closed on wrong-host reuse.** If the caller reuses this
transport for a DIFFERENT hostname (stored in a dict keyed by
origin, for example), we raise instead of falling through to an
unpinned ``connect_tcp`` — that fall-through is exactly the
TOCTOU the pin exists to close. Build a new transport per host.
"""
def __init__(self, *, hostname: str, resolved_ip: str) -> None:
super().__init__()
self._hostname = _normalize_pin_host(hostname)
self._resolved_ip = resolved_ip
def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: Iterable[SOCKET_OPTION] | None = None,
) -> Any:
normalized = _normalize_pin_host(host)
if normalized != self._hostname:
raise RuntimeError(
f"IpPinnedTransport is pinned to {self._hostname!r}; "
f"refusing connect to {host!r} — build a new transport per host "
f"(see build_ip_pinned_transport)"
)
return super().connect_tcp(
host=self._resolved_ip,
port=port,
timeout=timeout,
local_address=local_address,
socket_options=socket_options,
)
class _IpPinnedAsyncBackend(_AnyIOBackend):
"""Async counterpart to :class:`_IpPinnedSyncBackend`.
See :class:`_IpPinnedSyncBackend` for the fail-closed contract
on wrong-host reuse.
"""
def __init__(self, *, hostname: str, resolved_ip: str) -> None:
super().__init__()
self._hostname = _normalize_pin_host(hostname)
self._resolved_ip = resolved_ip
async def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: Iterable[SOCKET_OPTION] | None = None,
) -> Any:
normalized = _normalize_pin_host(host)
if normalized != self._hostname:
raise RuntimeError(
f"AsyncIpPinnedTransport is pinned to {self._hostname!r}; "
f"refusing connect to {host!r} — build a new transport per host "
f"(see abuild_ip_pinned_transport)"
)
return await super().connect_tcp(
host=self._resolved_ip,
port=port,
timeout=timeout,
local_address=local_address,
socket_options=socket_options,
)
class IpPinnedTransport(httpx.HTTPTransport):
"""``httpx.HTTPTransport`` that connects by pre-resolved IP.
Preserves normal httpx ergonomics — pass this to
``httpx.Client(transport=...)`` and everything else works
unchanged. The TLS handshake uses the original hostname for
SNI + cert validation; only the TCP destination is rewritten.
Construct via :func:`build_ip_pinned_transport` unless you've
already resolved the hostname yourself.
"""
def __init__(
self,
*,
hostname: str,
resolved_ip: str,
verify: bool = True,
retries: int = 0,
max_connections: int | None = 100,
max_keepalive_connections: int | None = 20,
) -> None:
if verify:
ssl_context = _build_ssl_context()
else:
warnings.warn(
"IpPinnedTransport constructed with verify=False — TLS cert "
"validation is disabled. Use only for tests against local "
"origins; NEVER in production.",
stacklevel=2,
)
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
backend = _IpPinnedSyncBackend(hostname=hostname, resolved_ip=resolved_ip)
# Build the ConnectionPool ourselves (rather than super().__init__
# and then reassign ._pool) so the TLS + backend config is set
# up atomically and we don't briefly own a vanilla pool. Match
# httpx's default connection limits explicitly — httpcore's
# ConnectionPool default is 10/_ which would be a surprise
# downgrade for callers who expect httpx-shaped pool sizing.
self._pool = httpcore.ConnectionPool(
ssl_context=ssl_context,
network_backend=backend,
http1=True,
http2=False,
retries=retries,
max_connections=max_connections,
max_keepalive_connections=max_keepalive_connections,
)
class AsyncIpPinnedTransport(httpx.AsyncHTTPTransport):
"""Async counterpart to :class:`IpPinnedTransport`."""
def __init__(
self,
*,
hostname: str,
resolved_ip: str,
verify: bool = True,
retries: int = 0,
max_connections: int | None = 100,
max_keepalive_connections: int | None = 20,
) -> None:
if verify:
ssl_context = _build_ssl_context()
else:
warnings.warn(
"AsyncIpPinnedTransport constructed with verify=False — TLS "
"cert validation is disabled. Use only for tests against "
"local origins; NEVER in production.",
stacklevel=2,
)
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
backend = _IpPinnedAsyncBackend(hostname=hostname, resolved_ip=resolved_ip)
self._pool = httpcore.AsyncConnectionPool(
ssl_context=ssl_context,
network_backend=backend,
http1=True,
http2=False,
retries=retries,
max_connections=max_connections,
max_keepalive_connections=max_keepalive_connections,
)
def build_ip_pinned_transport(
uri: str,
*,
allow_private: bool = False,
allowed_ports: frozenset[int] | None = None,
verify: bool = True,
) -> IpPinnedTransport:
"""Resolve ``uri`` once and return a transport pinned to the validated IP.
Raises :class:`SSRFValidationError` if the URI's scheme isn't
``http``/``https``, ``allowed_ports`` is set and the URI's port is
outside it, the host doesn't resolve, or every resolved IP is in a
blocked range.
``allowed_ports`` defaults to ``None`` (no port filter — AdCP
doesn't constrain webhook ports). Hardened deployments pass
:data:`adcp.signing.jwks.DEFAULT_ALLOWED_PORTS` (`{443, 8443}`)
or a custom set.
Typical use inside a fetcher::
transport = build_ip_pinned_transport(uri)
with httpx.Client(transport=transport, timeout=10.0) as client:
response = client.get(uri)
"""
hostname, resolved_ip, _port = resolve_and_validate_host(
uri,
allow_private=allow_private,
allowed_ports=allowed_ports,
)
return IpPinnedTransport(hostname=hostname, resolved_ip=resolved_ip, verify=verify)
def build_async_ip_pinned_transport(
uri: str,
*,
allow_private: bool = False,
allowed_ports: frozenset[int] | None = None,
verify: bool = True,
) -> AsyncIpPinnedTransport:
"""Build an :class:`AsyncIpPinnedTransport` for ``uri``.
Resolve + validate run synchronously (``socket.getaddrinfo``); this
function itself is not awaitable. The returned transport plugs
into :class:`httpx.AsyncClient`.
``allowed_ports`` defaults to ``None`` (no port filter); see
:func:`build_ip_pinned_transport` for the hardening kwarg
semantics.
"""
hostname, resolved_ip, _port = resolve_and_validate_host(
uri,
allow_private=allow_private,
allowed_ports=allowed_ports,
)
return AsyncIpPinnedTransport(hostname=hostname, resolved_ip=resolved_ip, verify=verify)
def abuild_ip_pinned_transport(
uri: str,
*,
allow_private: bool = False,
allowed_ports: frozenset[int] | None = None,
verify: bool = True,
) -> AsyncIpPinnedTransport:
"""Deprecated alias for :func:`build_async_ip_pinned_transport`.
The ``a``-prefix convention in this package means "awaitable
coroutine" (``averify_detached_jws`` etc.) — but this factory is
synchronous. Renamed during PR #206 review; kept for one release
so downstream callers have time to migrate.
"""
warnings.warn(
"abuild_ip_pinned_transport is deprecated; use "
"build_async_ip_pinned_transport (factory is sync, returns "
"an AsyncIpPinnedTransport).",
DeprecationWarning,
stacklevel=2,
)
return build_async_ip_pinned_transport(
uri,
allow_private=allow_private,
allowed_ports=allowed_ports,
verify=verify,
)