added specific error messages when RequestBlocked is raised, despite using a proxy

Author: jdepoix

README.md

@@ -284,7 +284,8 @@ therefore integrated it into this module, to make setting it up as easy as possi
 
 Once you have created a [Webshare account](https://www.webshare.io/?referral_code=w0xno53eb50g) and purchased a 
 "Residential" proxy package that suits your workload (make sure NOT to purchase "Proxy Server" or 
-"Static Residential"!), open the [Webshare Proxy Settings](https://dashboard.webshare.io/proxy/settings) to retrieve 
+"Static Residential"!), open the 
+[Webshare Proxy Settings](https://dashboard.webshare.io/proxy/settings?referral_code=w0xno53eb50g) to retrieve 
 your "Proxy Username" and "Proxy Password". Using this information you can initialize the `YouTubeTranscriptApi` as 
 follows:
 
@@ -306,8 +307,8 @@ ytt_api.fetch(video_id)
 Using the `WebshareProxyConfig` will default to using rotating residential proxies and requires no further 
 configuration.
 
-Note that referral links are used here and any purchases made through these links will support this Open Source 
-project, which is very much appreciated! 💖😊🙏💖
+Note that [referral links are used here](https://www.webshare.io/?referral_code=w0xno53eb50g) and any purchases 
+made through these links will support this Open Source project, which is very much appreciated! 💖😊🙏💖
 
 However, you are of course free to integrate your own proxy solution using the `GenericProxyConfig` class, if you 
 prefer using another provider or want to implement your own solution, as covered by the following section.
@@ -511,7 +512,7 @@ using residential proxies as explained in
 create a [Webshare account](https://www.webshare.io/?referral_code=w0xno53eb50g) and purchase a "Residential" proxy 
 package that suits your workload (make sure NOT to purchase "Proxy Server" or "Static Residential"!). Then you can use 
 the "Proxy Username" and "Proxy Password" which you can find in your 
-[Webshare Proxy Settings](https://dashboard.webshare.io/proxy/settings), to run the following command:
+[Webshare Proxy Settings](https://dashboard.webshare.io/proxy/settings?referral_code=w0xno53eb50g), to run the following command:
 
 ```
 youtube_transcript_api <first_video_id> <second_video_id> --webshare-proxy-username "username" --webshare-proxy-password "password"

youtube_transcript_api/_errors.py

@@ -4,6 +4,7 @@
 from requests import HTTPError
 
 from ._settings import WATCH_URL
+from .proxies import ProxyConfig, GenericProxyConfig, WebshareProxyConfig
 
 
 class YouTubeTranscriptApiException(Exception):
@@ -45,7 +46,7 @@ class CouldNotRetrieveTranscript(YouTubeTranscriptApiException):
 
     def __init__(self, video_id: str):
         self.video_id = video_id
-        super().__init__(self._build_error_message())
+        super().__init__()
 
     def _build_error_message(self) -> str:
         error_message = self.ERROR_MESSAGE.format(
@@ -64,6 +65,9 @@ def _build_error_message(self) -> str:
     def cause(self) -> str:
         return self.CAUSE_MESSAGE
 
+    def __str__(self) -> str:
+        return self._build_error_message()
+
 
 class YouTubeRequestFailed(CouldNotRetrieveTranscript):
     CAUSE_MESSAGE = "Request to YouTube failed: {reason}"
@@ -135,6 +139,51 @@ class RequestBlocked(CouldNotRetrieveTranscript):
         "eventually permanently ban the account that you have used to authenticate "
         "with! So only do this if you don't mind your account being banned!"
     )
+    WITH_GENERIC_PROXY_CAUSE_MESSAGE = (
+        "YouTube is blocking your requests, despite you using proxies. Keep in mind "
+        "a proxy is just a way to hide your real IP behind the IP of that proxy, but "
+        "there is no guarantee that the IP of that proxy won't be blocked as well.\n\n"
+        "The only truly reliable way to prevent IP blocks is rotating through a large "
+        "pool of residential IPs, by using a provider like Webshare "
+        "(https://www.webshare.io/?referral_code=w0xno53eb50g), which provides you "
+        "with a pool of >30M residential IPs (make sure to purchase "
+        '"Residential" proxies, NOT "Proxy Server" or "Static Residential"!).\n\n'
+        "You will find more information on how to easily integrate Webshare here: "
+        "https://github.com/jdepoix/youtube-transcript-api"
+        "?tab=readme-ov-file#using-webshare"
+    )
+    WITH_WEBSHARE_PROXY_CAUSE_MESSAGE = (
+        "YouTube is blocking your requests, despite you using Webshare proxies. "
+        'Please make sure that you have purchased "Residential" proxies and '
+        'NOT "Proxy Server" or "Static Residential", as those won\'t work as '
+        'reliably! The free tier also uses "Proxy Server" and will NOT work!\n\n'
+        'The only reliable option is using "Residential" proxies (not "Static '
+        'Residential"), as this allows you to rotate through a pool of over 30M IPs, '
+        "which means you will always find an IP that hasn't been blocked by YouTube "
+        "yet!\n\n"
+        "You can support the development of this open source project by making your "
+        "Webshare purchases through this affiliate link: "
+        "https://www.webshare.io/?referral_code=w0xno53eb50g \n\n"
+        "Thank you for your support! <3"
+    )
+
+    def __init__(self, video_id: str):
+        self._proxy_config = None
+        super().__init__(video_id)
+
+    def with_proxy_config(
+        self, proxy_config: Optional[ProxyConfig]
+    ) -> "RequestBlocked":
+        self._proxy_config = proxy_config
+        return self
+
+    @property
+    def cause(self) -> str:
+        if isinstance(self._proxy_config, WebshareProxyConfig):
+            return self.WITH_WEBSHARE_PROXY_CAUSE_MESSAGE
+        if isinstance(self._proxy_config, GenericProxyConfig):
+            return self.WITH_GENERIC_PROXY_CAUSE_MESSAGE
+        return super().cause
 
 
 class IpBlocked(RequestBlocked):

youtube_transcript_api/_transcripts.py

@@ -364,7 +364,7 @@ def _fetch_captions_json(self, video_id: str, try_number: int = 0) -> Dict:
             )
             if try_number + 1 < retries:
                 return self._fetch_captions_json(video_id, try_number=try_number + 1)
-            raise exception
+            raise exception.with_proxy_config(self._proxy_config)
 
     def _extract_captions_json(self, html: str, video_id: str) -> Dict:
         splitted_html = html.split("var ytInitialPlayerResponse = ")

youtube_transcript_api/test/test_api.py

@@ -247,9 +247,11 @@ def test_fetch__exception_if_youtube_request_fails(self):
             httpretty.GET, "https://www.youtube.com/watch", status=500
         )
 
-        with self.assertRaises(YouTubeRequestFailed):
+        with self.assertRaises(YouTubeRequestFailed) as cm:
             YouTubeTranscriptApi().fetch("abc")
 
+        self.assertIn("Request to YouTube failed: ", str(cm.exception))
+
     def test_fetch__exception_if_age_restricted(self):
         httpretty.register_uri(
             httpretty.GET,
@@ -277,21 +279,24 @@ def test_fetch__exception_request_blocked(self):
             body=load_asset("youtube_request_blocked.html.static"),
         )
 
-        with self.assertRaises(RequestBlocked):
+        with self.assertRaises(RequestBlocked) as cm:
             YouTubeTranscriptApi().fetch("Njp5uhTorCo")
 
+        self.assertIn("YouTube is blocking requests from your IP", str(cm.exception))
+
     def test_fetch__exception_unplayable(self):
         httpretty.register_uri(
             httpretty.GET,
             "https://www.youtube.com/watch",
             body=load_asset("youtube_unplayable.html.static"),
         )
 
-        with self.assertRaises(VideoUnplayable) as error:
+        with self.assertRaises(VideoUnplayable) as cm:
             YouTubeTranscriptApi().fetch("Njp5uhTorCo")
-        error = error.exception
-        self.assertEqual(error.reason, "Custom Reason")
-        self.assertEqual(error.sub_reasons, ["Sub Reason 1", "Sub Reason 2"])
+        exception = cm.exception
+        self.assertEqual(exception.reason, "Custom Reason")
+        self.assertEqual(exception.sub_reasons, ["Sub Reason 1", "Sub Reason 2"])
+        self.assertIn("Custom Reason", str(exception))
 
     def test_fetch__exception_if_transcripts_disabled(self):
         httpretty.register_uri(
@@ -312,9 +317,11 @@ def test_fetch__exception_if_transcripts_disabled(self):
             YouTubeTranscriptApi().fetch("Fjg5lYqvzUs")
 
     def test_fetch__exception_if_language_unavailable(self):
-        with self.assertRaises(NoTranscriptFound):
+        with self.assertRaises(NoTranscriptFound) as cm:
             YouTubeTranscriptApi().fetch("GJLlxj_dtq8", languages=["cz"])
 
+        self.assertIn("No transcripts were found for", str(cm.exception))
+
     @patch("youtube_transcript_api.proxies.GenericProxyConfig.to_requests_dict")
     def test_fetch__with_proxy(self, to_requests_dict):
         proxy_config = GenericProxyConfig(
@@ -359,7 +366,7 @@ def test_fetch__with_proxy_retry_when_blocked(self, to_requests_dict):
         self.assertEqual(len(httpretty.latest_requests()), 3 + 2)
 
     @patch("youtube_transcript_api.proxies.GenericProxyConfig.to_requests_dict")
-    def test_fetch__with_proxy_reraise_when_blocked(self, to_requests_dict):
+    def test_fetch__with_webshare_proxy_reraise_when_blocked(self, to_requests_dict):
         retries = 5
         for _ in range(retries):
             httpretty.register_uri(
@@ -373,10 +380,31 @@ def test_fetch__with_proxy_reraise_when_blocked(self, to_requests_dict):
             retries_when_blocked=retries,
         )
 
-        with self.assertRaises(RequestBlocked):
+        with self.assertRaises(RequestBlocked) as cm:
             YouTubeTranscriptApi(proxy_config=proxy_config).fetch("Njp5uhTorCo")
 
         self.assertEqual(len(httpretty.latest_requests()), retries)
+        self.assertEqual(cm.exception._proxy_config, proxy_config)
+        self.assertIn("Webshare", str(cm.exception))
+
+    @patch("youtube_transcript_api.proxies.GenericProxyConfig.to_requests_dict")
+    def test_fetch__with_generic_proxy_reraise_when_blocked(self, to_requests_dict):
+        httpretty.register_uri(
+            httpretty.GET,
+            "https://www.youtube.com/watch",
+            body=load_asset("youtube_request_blocked.html.static"),
+        )
+        proxy_config = GenericProxyConfig(
+            http_url="http://localhost:8080",
+            https_url="http://localhost:8080",
+        )
+
+        with self.assertRaises(RequestBlocked) as cm:
+            YouTubeTranscriptApi(proxy_config=proxy_config).fetch("Njp5uhTorCo")
+
+        self.assertEqual(len(httpretty.latest_requests()), 1)
+        self.assertEqual(cm.exception._proxy_config, proxy_config)
+        self.assertIn("YouTube is blocking your requests", str(cm.exception))
 
     def test_fetch__with_cookies(self):
         cookie_path = get_asset_path("example_cookies.txt")