from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import json
import time

main_url = "https://news.google.com"

# Set up Chrome options to enable performance logging
chrome_options = Options()

# --- Disable image loading ---
prefs = {"profile.managed_default_content_settings.images": 2} 
chrome_options.add_experimental_option("prefs", prefs)

chrome_options.set_capability("goog:loggingPrefs", {"performance": "ALL"})

# Initialize the WebDriver with the specified options
driver = webdriver.Chrome(options=chrome_options)

# Navigate to the target webpage
driver.get(main_url)

time.sleep(6)  # Increased wait time to ensure API calls are made

# Retrieve the performance logs
logs = driver.get_log("performance")

# Process and extract relevant network information
for entry in logs:
    log = json.loads(entry["message"])["message"]
    if log["method"] == "Network.responseReceived":
        # Get the URL from the response parameters
        url = log['params']['response']['url']

        # Only proceed if the URL contains "/api" ---
        if "/api" in url:
            print(f"URL: {url}")
            print(f"Status: {log['params']['response']['status']}")

            # To get the response body, you need the requestId
            request_id = log["params"]["requestId"]
            try:
                response_body = driver.execute_cdp_cmd(
                    "Network.getResponseBody", {"requestId": request_id}
                )

                # Check if the response body is base64 encoded
                if response_body.get("base64Encoded", False):
                    print("*Base64 content was found, not printing out *")
                else:
                    body_content = response_body.get('body', '')
                    if body_content: # Ensure content exists before slicing 350
                        print(f"Response Body: {body_content[:350]}")
                    else:
                        print("Response Body: (empty)")

            except Exception as e:
                print(f"Could not get response body for {request_id}: {e}")

            # Add a separator for clarity between matching API calls
            print("-" * 52)

# Quit the driver
driver.quit()