|
| 1 | +from fileinput import filename |
| 2 | +import sys |
| 3 | +import time |
| 4 | +import json |
| 5 | +from typing import List |
| 6 | +from bs4 import BeautifulSoup |
| 7 | +from selenium import webdriver |
| 8 | +from selenium.webdriver.chrome.service import Service |
| 9 | +from selenium.webdriver.common.by import By |
| 10 | +from selenium.common.exceptions import StaleElementReferenceException |
| 11 | +from chromedriver_autoinstaller import install as install_chromedriver |
| 12 | + |
| 13 | +def find_load_more_button(driver): |
| 14 | + """ |
| 15 | + Find the Load More button by searching through all buttons and examining their text content. |
| 16 | + Returns the button element if found, None otherwise. |
| 17 | + """ |
| 18 | + try: |
| 19 | + # Find all buttons on the page |
| 20 | + buttons = driver.find_elements(By.TAG_NAME, "button") |
| 21 | + |
| 22 | + for button in buttons: |
| 23 | + if button and button.is_displayed(): |
| 24 | + try: |
| 25 | + # Get the text content of the button including all nested elements |
| 26 | + button_text = driver.execute_script(""" |
| 27 | + function getTextContent(element) { |
| 28 | + // Get text from the element itself |
| 29 | + let text = element.textContent || element.innerText || ''; |
| 30 | +
|
| 31 | + // Also check all child elements for text |
| 32 | + const children = element.querySelectorAll('*'); |
| 33 | + for (let child of children) { |
| 34 | + if (child.textContent) { |
| 35 | + text += ' ' + child.textContent; |
| 36 | + } |
| 37 | + } |
| 38 | +
|
| 39 | + return text.trim().toLowerCase(); |
| 40 | + } |
| 41 | + return getTextContent(arguments[0]); |
| 42 | + """, button) |
| 43 | + |
| 44 | + # Check if the button contains "load more" text |
| 45 | + if button_text and "load more" in button_text: |
| 46 | + return button |
| 47 | + |
| 48 | + except StaleElementReferenceException: |
| 49 | + continue |
| 50 | + |
| 51 | + return None |
| 52 | + |
| 53 | + except Exception as e: |
| 54 | + print(f"Error finding load more button: {e}") |
| 55 | + return None |
| 56 | + |
| 57 | +def get_image_links_selenium(url): |
| 58 | + """ |
| 59 | + Crawls a Pexels page using Selenium to fetch the links of all featured images. |
| 60 | + Auto-clicks the "Load More" button if exists to load more images. |
| 61 | + """ |
| 62 | + driver = None # Initialize driver to None |
| 63 | + try: |
| 64 | + # Automatically install and set up chromedriver |
| 65 | + service = Service(install_chromedriver()) |
| 66 | + |
| 67 | + # Set up Chrome options for headless mode |
| 68 | + options = webdriver.ChromeOptions() |
| 69 | + options.add_argument('--headless') |
| 70 | + options.add_argument('--disable-gpu') |
| 71 | + options.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36') |
| 72 | + |
| 73 | + driver = webdriver.Chrome(service=service, options=options) |
| 74 | + |
| 75 | + print("Fetching URL with Selenium...") |
| 76 | + driver.get(url) |
| 77 | + # Wait for initial page load |
| 78 | + time.sleep(3) |
| 79 | + |
| 80 | + max_clicks = 5 # Safety limit to prevent infinite loops |
| 81 | + click_count = 0 |
| 82 | + |
| 83 | + # Try clicking the Load More button |
| 84 | + while click_count < max_clicks: |
| 85 | + load_more_button = find_load_more_button(driver) |
| 86 | + |
| 87 | + if load_more_button and load_more_button.is_displayed(): |
| 88 | + try: |
| 89 | + # Click the button |
| 90 | + print(f"Found Load More button, clicking... (attempt {click_count + 1})") |
| 91 | + click_count += 1 |
| 92 | + load_more_button.click() |
| 93 | + |
| 94 | + except (StaleElementReferenceException, Exception) as e: |
| 95 | + # print(f"Error clicking button: {e}") |
| 96 | + continue |
| 97 | + else: |
| 98 | + print("No Load More button left to click.") |
| 99 | + break |
| 100 | + |
| 101 | + print(f"Finished clicking Load More buttons. Total clicks: {click_count}") |
| 102 | + |
| 103 | + # Wait a bit more to ensure all content is loaded |
| 104 | + time.sleep(2) |
| 105 | + |
| 106 | + html = driver.page_source |
| 107 | + soup = BeautifulSoup(html, 'html.parser') |
| 108 | + |
| 109 | + imgs = soup.find_all('img') |
| 110 | + img_links: List[str] = [] |
| 111 | + for img in imgs: |
| 112 | + src = img.get('src', '') |
| 113 | + if isinstance(src, str) and src.startswith('https://images.pexels.com/photos/'): |
| 114 | + img_links.append(src) |
| 115 | + |
| 116 | + processed_img_links: List[str] = [] |
| 117 | + for link in img_links: |
| 118 | + processed_link = link.split('?')[0] |
| 119 | + if processed_link not in processed_img_links: |
| 120 | + processed_img_links.append(processed_link) |
| 121 | + |
| 122 | + return processed_img_links |
| 123 | + |
| 124 | + except Exception as e: |
| 125 | + print(f"An error occurred: {e}") |
| 126 | + return [] |
| 127 | + finally: |
| 128 | + if driver: |
| 129 | + driver.quit() |
| 130 | + |
| 131 | +def save_links_to_json(links: List[str], filename: str): |
| 132 | + """ |
| 133 | + Save the list of image links to a JSON file. |
| 134 | + """ |
| 135 | + try: |
| 136 | + with open(filename, 'w') as f: |
| 137 | + json.dump({ |
| 138 | + "images": links |
| 139 | + }, f, indent=4) |
| 140 | + print(f"Image links saved to {filename}") |
| 141 | + except Exception as e: |
| 142 | + print(f"Error saving links to JSON: {e}") |
| 143 | + |
| 144 | +if __name__ == "__main__": |
| 145 | + pexels_url = "https://www.pexels.com/@perry-z-1662054943/featured-uploads/" |
| 146 | + links = get_image_links_selenium(pexels_url) |
| 147 | + |
| 148 | + if links: |
| 149 | + print("\nFound the following image links:") |
| 150 | + print(json.dumps(links, indent=4)) |
| 151 | + print(f"\nTotal image links found: {len(links)}") |
| 152 | + if sys.argv and len(sys.argv) > 1: |
| 153 | + output_filename = sys.argv[1] |
| 154 | + save_links_to_json(links, output_filename) |
| 155 | + else: |
| 156 | + print("\nNo image links were found.") |
0 commit comments