Merge pull request #296 from kollivier/download_fixes

Fix issues with relative CSS url() detection…

Author: kollivier

ricecooker/config.py

@@ -21,6 +21,12 @@
 SUSHI_BAR_CLIENT = None
 STAGE = False
 
+# When this is set to true, any failure will raise an error and stop the chef.
+# This will likely be set to true in a future version of ricecooker, once
+# we can ensure all ricecooker internal functions handle non-fatal errors
+# properly.
+STRICT = False
+
 # Sometimes chef runs will get stuck indefinitely waiting on data from SSL conn,
 # so we add a timeout value as suggested in https://stackoverflow.com/a/30771995
 socket.setdefaulttimeout(20)

ricecooker/utils/downloader.py

@@ -11,7 +11,7 @@
 from selenium import webdriver
 import selenium.webdriver.support.ui as selenium_ui
 from requests_file import FileAdapter
-from ricecooker.config import LOGGER, PHANTOMJS_PATH
+from ricecooker.config import LOGGER, PHANTOMJS_PATH, STRICT
 from ricecooker.utils.html import download_file
 from ricecooker.utils.caching import CacheForeverHeuristic, FileCache, CacheControlAdapter, InvalidatingCacheControlAdapter
 
@@ -146,6 +146,8 @@ def make_request(url, clear_cookies=False, headers=None, timeout=60, *args, **kw
 
     if response.status_code != 200:
         print("NOT FOUND:", url)
+        if STRICT:
+            response.raise_for_status()
 
     return response
 
@@ -235,17 +237,30 @@ def css_content_middleware(content, url, **kwargs):
         if css_middleware:
             content = css_middleware(content, url, **kwargs)
 
-        file_dir = os.path.dirname(urlparse(url).path)
+        root_parts = urlparse(url)
 
         # Download linked fonts and images
         def repl(match):
             src = match.group(1)
+
             if src.startswith('//localhost'):
                 return 'url()'
             # Don't download data: files
             if src.startswith('data:'):
                 return match.group(0)
-            src_url = urljoin(base_url, os.path.join(file_dir, src))
+            parts = urlparse(src)
+            root_url = None
+            if url:
+                root_url = url[:url.rfind('/') + 1]
+
+            if parts.scheme and parts.netloc:
+                src_url = src
+            elif parts.path.startswith('/') and url:
+                src_url = '{}://{}{}'.format(root_parts.scheme, root_parts.netloc, root_parts.path)
+            elif url and root_url:
+                src_url = urljoin(root_url, src)
+            else:
+                src_url = urljoin(base_url, src)
 
             if _is_blacklisted(src_url, url_blacklist):
                 print('        Skipping downloading blacklisted url', src_url)

ricecooker/utils/html.py

@@ -9,7 +9,7 @@
 from urllib.parse import urlparse, unquote
 
 from .caching import FileCache, CacheControlAdapter
-from ricecooker.config import PHANTOMJS_PATH
+from ricecooker.config import LOGGER, PHANTOMJS_PATH, STRICT
 
 
 
@@ -99,6 +99,11 @@ def download_file(url, destpath, filename=None, baseurl=None, subpath=None, midd
     response = request_fn(url)
     content = response.content
 
+    if STRICT:
+        response.raise_for_status()
+    elif response.status_code >= 400:
+        LOGGER.warning("URL {} returned status {}".format(url, response.status_code))
+
     # if there are any middleware callbacks, apply them to the content
     if middleware_callbacks:
         content = content.decode()