Skip to content

Commit 35032d6

Browse files
committed
fix: solving a bug with using configure on Fetcher
1 parent 54a080a commit 35032d6

1 file changed

Lines changed: 46 additions & 9 deletions

File tree

scrapling/fetchers/requests.py

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,65 @@
1+
from scrapling.core._types import Any, Awaitable, Unpack
2+
from scrapling.engines._browsers._types import DataRequestParams, GetRequestParams
13
from scrapling.engines.static import (
24
FetcherSession,
35
FetcherClient as _FetcherClient,
46
AsyncFetcherClient as _AsyncFetcherClient,
57
)
6-
from scrapling.engines.toolbelt.custom import BaseFetcher
8+
from scrapling.engines.toolbelt.custom import BaseFetcher, Response
9+
10+
__all__ = ["Fetcher", "AsyncFetcher", "FetcherSession"]
711

812

913
__FetcherClientInstance__ = _FetcherClient()
1014
__AsyncFetcherClientInstance__ = _AsyncFetcherClient()
1115

1216

17+
def _merge_selector_config(cls: type[BaseFetcher], kwargs: Any) -> Any:
18+
"""Merge class-level parser arguments into per-request ``selector_config``.
19+
20+
Values from ``Fetcher.configure(...)`` act as the base; any explicit
21+
``selector_config`` passed on the call overrides them.
22+
"""
23+
selector_config = kwargs.get("selector_config") or {}
24+
kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
25+
return kwargs
26+
27+
1328
class Fetcher(BaseFetcher):
1429
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
1530

16-
get = __FetcherClientInstance__.get
17-
post = __FetcherClientInstance__.post
18-
put = __FetcherClientInstance__.put
19-
delete = __FetcherClientInstance__.delete
31+
@classmethod
32+
def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Response:
33+
return __FetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs))
34+
35+
@classmethod
36+
def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:
37+
return __FetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs))
38+
39+
@classmethod
40+
def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:
41+
return __FetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs))
42+
43+
@classmethod
44+
def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:
45+
return __FetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs))
2046

2147

2248
class AsyncFetcher(BaseFetcher):
2349
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
2450

25-
get = __AsyncFetcherClientInstance__.get
26-
post = __AsyncFetcherClientInstance__.post
27-
put = __AsyncFetcherClientInstance__.put
28-
delete = __AsyncFetcherClientInstance__.delete
51+
@classmethod
52+
def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Awaitable[Response]:
53+
return __AsyncFetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs))
54+
55+
@classmethod
56+
def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:
57+
return __AsyncFetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs))
58+
59+
@classmethod
60+
def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:
61+
return __AsyncFetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs))
62+
63+
@classmethod
64+
def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:
65+
return __AsyncFetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs))

0 commit comments

Comments
 (0)