Wrestling Undetected-Chromedriver & Persistence in NixOS

[mike@nixos:~/repos/pipulate]$ git --no-pager diff
diff --git a/pipulate/core.py b/pipulate/core.py
index 9a2d9a54..f06cc4b0 100644
--- a/pipulate/core.py
+++ b/pipulate/core.py
@@ -1881,37 +1881,39 @@ class Pipulate:
         state = self.read_state(job)
         return state.get(step, default)
 
-    # START: scrape_method
-    async def scrape(self, url: str, take_screenshot: bool = False, mode: str = 'selenium', headless: bool = True, verbose: bool = True, **kwargs):
+    # Add persistent=False and profile_name=None to the function signature
+    async def scrape(self, url: str, take_screenshot: bool = False, mode: str = 'selenium', headless: bool = True, verbose: bool = True, persistent: bool = False, profile_name: str = "default", **kwargs):
         """
         Gives AI "eyes" by performing browser automation or HTTP requests to scrape a URL.
-    
+
         This method is the primary entrypoint for scraping and supports multiple modes.
         The default mode is 'selenium' which uses a full browser.
-    
+
         Args:
             url (str): The URL to scrape.
             take_screenshot (bool): Whether to capture a screenshot (selenium mode only). Defaults to False.
             mode (str): The scraping mode to use ('selenium', 'requests', etc.). Defaults to 'selenium'.
             headless (bool): Whether to run the browser in headless mode (selenium mode only). Defaults to True.
+            persistent (bool): Whether to use a persistent browser profile. Defaults to False.
+            profile_name (str): The name of the persistent profile to use. Defaults to "default".
             **kwargs: Additional parameters to pass to the underlying automation tool.
-    
+
         Returns:
             dict: The result from the scraper tool, including paths to captured artifacts.
         """
         from tools.scraper_tools import selenium_automation
         from urllib.parse import urlparse, quote
         from datetime import datetime
-    
-        logger.info(f"👁️‍🗨️ Initiating scrape for: {url} (Mode: {mode}, Headless: {headless})")
-    
+
+        logger.info(f"👁️‍🗨️ Initiating scrape for: {url} (Mode: {mode}, Headless: {headless}, Persistent: {persistent})")
+
         # --- New Directory Logic ---
         parsed_url = urlparse(url)
         domain = parsed_url.netloc
         path = parsed_url.path or '/'
         # Use quote with an empty safe string to encode everything, including slashes
         url_path_slug = quote(path, safe='')
-    
+
         params = {
             "url": url,
             "domain": domain,
@@ -1920,9 +1922,11 @@ class Pipulate:
             "headless": headless,
             "is_notebook_context": self.is_notebook_context, # Pass the context flag
             "verbose": verbose,
+            "persistent": persistent,
+            "profile_name": profile_name,
             **kwargs # Pass through any other params
         }
-    
+
         if mode == 'selenium':
             try:
                 result = await selenium_automation(params)
@@ -1933,8 +1937,6 @@ class Pipulate:
         else:
             logger.warning(f"Scrape mode '{mode}' is not yet implemented.")
             return {"success": False, "error": f"Mode '{mode}' not implemented."}
-    # END: scrape_method
-
 
     def _find_project_root(self, start_path):
         """Walks up from a starting path to find the project root (marked by 'flake.nix')."""
diff --git a/requirements.in b/requirements.in
index b8bd294c..c821ea90 100644
--- a/requirements.in
+++ b/requirements.in
@@ -41,8 +41,8 @@ uvicorn
 selenium
 selenium-wire
 selenium-stealth
-undetected-chromedriver
 webdriver-manager
+undetected-chromedriver
 
 # Data Science and Plotting
 matplotlib
diff --git a/tools/scraper_tools.py b/tools/scraper_tools.py
index 3644f732..70f9d7e4 100644
--- a/tools/scraper_tools.py
+++ b/tools/scraper_tools.py
@@ -1,21 +1,19 @@
 # /home/mike/repos/pipulate/tools/scraper_tools.py
+import faulthandler
+faulthandler.enable()
+
 import asyncio
 import json
 import os
 import sys
 import shutil
+import tempfile
 from datetime import datetime
 from pathlib import Path
 from urllib.parse import quote, urlparse
 
 from loguru import logger
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.chrome.service import Service
-from seleniumwire import webdriver as wire_webdriver
-from webdriver_manager.chrome import ChromeDriverManager
-from selenium_stealth import stealth
-from selenium import webdriver
-
+import undetected_chromedriver as uc
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
@@ -37,28 +35,27 @@ def get_safe_path_component(url: str) -> tuple[str, str]:
 @auto_tool
 async def selenium_automation(params: dict) -> dict:
     """
-    Performs an advanced browser automation scrape of a single URL.
-    Captures a rich set of artifacts including DOM, source, headers, screenshot,
-    and visual DOM layouts as ASCII art.
+    Performs an advanced browser automation scrape of a single URL using undetected-chromedriver.
+    Supports persistent profiles to maintain sessions across runs.
+    Captures artifacts including DOM, source, headers, screenshot, and visual DOM layouts.
     """
-    verbose = params.get("verbose", True)
     url = params.get("url")
     domain = params.get("domain")
     url_path_slug = params.get("url_path_slug")
     take_screenshot = params.get("take_screenshot", False)
     headless = params.get("headless", True)
     is_notebook_context = params.get("is_notebook_context", False)
-
-    options = webdriver.ChromeOptions()
-    options.add_experimental_option("excludeSwitches", ["enable-automation"])
-    options.add_experimental_option('useAutomationExtension', False)
-
+    persistent = params.get("persistent", False)
+    profile_name = params.get("profile_name", "default")
+    verbose = params.get("verbose", True)
 
     if not all([url, domain, url_path_slug is not None]):
         return {"success": False, "error": "URL, domain, and url_path_slug parameters are required."}
 
     driver = None
     artifacts = {}
+    profile_path = None
+    temp_profile = False
 
     base_dir = Path("browser_cache/")
     if not is_notebook_context:
@@ -68,142 +65,92 @@ async def selenium_automation(params: dict) -> dict:
 
     try:
         if output_dir.exists():
-            logger.info(f"🗑️ Clearing existing artifacts in: {output_dir}")
+            if verbose: logger.info(f"🗑️ Clearing existing artifacts in: {output_dir}")
             shutil.rmtree(output_dir)
         output_dir.mkdir(parents=True, exist_ok=True)
-        logger.info(f"💾 Saving new artifacts to: {output_dir}")
+        if verbose: logger.info(f"💾 Saving new artifacts to: {output_dir}")
 
-        chrome_options = Options()
+        options = uc.ChromeOptions()
         if headless:
-            chrome_options.add_argument("--headless")
-        chrome_options.add_argument("--no-sandbox")
-        chrome_options.add_argument("--disable-dev-shm-usage")
-        chrome_options.add_argument("--start-maximized")
-        chrome_options.add_argument("--window-size=1920,1080")
-
-        effective_os = os.environ.get('EFFECTIVE_OS', sys.platform)
-        service = Service(ChromeDriverManager().install()) if effective_os == 'darwin' else Service()
-
-        logger.info(f"🚀 Initializing Chrome driver (Headless: {headless})...")
-        driver = wire_webdriver.Chrome(service=service, options=chrome_options)
-
-        stealth(driver,
-            languages=["en-US", "en"],
-            vendor="Google Inc.",
-            platform="Win32",
-            webgl_vendor="Intel Inc.",
-            renderer="Intel Iris OpenGL Engine",
-            fix_hairline=True,
-        )
-
-        # --- ADD THIS BLOCK TO MANUALLY HIDE THE WEBDRIVER FLAG ---
-        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
-            "source": "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
-        })
+            options.add_argument("--headless")
+        options.add_argument("--no-sandbox")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("--start-maximized")
+        options.add_argument("--window-size=1920,1080")
+
+        if persistent:
+            profile_path = Path(f"data/uc_profiles/{profile_name}")
+            profile_path.mkdir(parents=True, exist_ok=True)
+            logger.info(f"🔒 Using persistent profile: {profile_path}")
+        else:
+            profile_path = tempfile.mkdtemp(prefix='pipulate_automation_')
+            temp_profile = True
+            logger.info(f"👻 Using temporary profile: {profile_path}")
+        
+        logger.info(f"🚀 Initializing undetected-chromedriver (Headless: {headless})...")
+        driver = uc.Chrome(options=options, user_data_dir=str(profile_path))
 
         logger.info(f"Navigating to: {url}")
         driver.get(url)
 
-
-
-        # --- NEW: Two-stage wait for challenge pages that reload ---
         try:
-            logger.info("Waiting for security challenge to trigger a reload (Stage 1)...")
-            # Grab a reference to the body of the initial challenge page
+            if verbose: logger.info("Waiting for security challenge to trigger a reload (Stage 1)...")
             initial_body = driver.find_element(By.TAG_NAME, 'body')
-
-            # Wait up to 20 seconds for the challenge to solve and location.reload() to be called.
-            # This is detected by waiting for the initial body element to go stale.
             WebDriverWait(driver, 20).until(EC.staleness_of(initial_body))
-            logger.success("✅ Page reload detected!")
-
-            logger.info("Waiting for main content to appear after reload (Stage 2)...")
-            # Now that the page has reloaded, wait up to 10 seconds for the actual content.
-            # Change 'main-content' to a selector that exists on your final target page.
-            WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "main-content")))
-            logger.success("✅ Main content located!")
-
+            if verbose: logger.success("✅ Page reload detected!")
+            
+            if verbose: logger.info("Waiting for main content to appear after reload (Stage 2)...")
+            WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "body")))
+            if verbose: logger.success("✅ Main content located!")
         except Exception as e:
-            logger.warning(f"Did not detect the expected page lifecycle for the security challenge. Proceeding anyway. Error: {e}")
-            # If the wait fails, we can still try to capture the page as-is.
-
-
+            if verbose: logger.warning(f"Did not detect a page reload for security challenge. Proceeding anyway. Error: {e}")
 
         # --- Capture Core Artifacts ---
         dom_path = output_dir / "rendered_dom.html"
         dom_path.write_text(driver.execute_script("return document.documentElement.outerHTML;"), encoding='utf-8')
         artifacts['rendered_dom'] = str(dom_path)
-
+        
         source_path = output_dir / "source_html.text"
         source_path.write_text(driver.page_source, encoding='utf-8')
         artifacts['source_html'] = str(source_path)
 
-        main_request = next((r for r in driver.requests if r.response and r.url == url), driver.last_request)
-        if main_request and main_request.response:
-            headers_path = output_dir / "response_headers.json"
-            headers_path.write_text(json.dumps(dict(main_request.response.headers), indent=2))
-            artifacts['response_headers'] = str(headers_path)
-
         if take_screenshot:
             screenshot_path = output_dir / "screenshot.png"
             driver.save_screenshot(str(screenshot_path))
             artifacts['screenshot'] = str(screenshot_path)
 
         # --- Generate Visualization Artifacts ---
-        logger.info(f"🎨 Generating DOM box visualization...")
-        viz_result = await dom_tools.visualize_dom_boxes({"file_path": str(dom_path), "verbose": verbose})
+        if verbose: logger.info(f"🎨 Generating DOM box visualization...")
+        viz_result = await dom_tools.visualize_dom_boxes({"file_path": str(dom_path), "verbose": False})
         if viz_result.get("success"):
             viz_path = output_dir / "dom_layout_boxes.txt"
             viz_path.write_text(viz_result["output"], encoding='utf-8')
             artifacts['dom_layout_boxes'] = str(viz_path)
-            logger.success("✅ DOM box layout saved.")
-        else:
-            logger.warning(f"⚠️ Could not generate DOM box visualization: {viz_result.get('error')}")
-            
-        logger.info(f"🌳 Generating DOM hierarchy visualization...")
-        hierarchy_viz_result = await dom_tools.visualize_dom_hierarchy({"file_path": str(dom_path), "verbose": verbose}) 
+        
+        if verbose: logger.info(f"🌳 Generating DOM hierarchy visualization...")
+        hierarchy_viz_result = await dom_tools.visualize_dom_hierarchy({"file_path": str(dom_path), "verbose": False})
         if hierarchy_viz_result.get("success"):
             hierarchy_viz_path = output_dir / "dom_hierarchy.txt"
             hierarchy_viz_path.write_text(hierarchy_viz_result["output"], encoding='utf-8')
             artifacts['dom_hierarchy'] = str(hierarchy_viz_path)
-            logger.success("✅ DOM hierarchy saved.")
-        else:
-            logger.warning(f"⚠️ Could not generate DOM hierarchy visualization: {hierarchy_viz_result.get('error')}")
-
-        # --- NEW: Generate Accessibility Tree Artifact ---
-        logger.info("🌲 Extracting accessibility tree...")
+            
+        # --- Generate Accessibility Tree Artifact ---
+        if verbose: logger.info("🌲 Extracting accessibility tree...")
         try:
             driver.execute_cdp_cmd("Accessibility.enable", {})
             ax_tree_result = driver.execute_cdp_cmd("Accessibility.getFullAXTree", {})
-            accessibility_tree = ax_tree_result.get("nodes", [])
-            
+            ax_tree = ax_tree_result.get("nodes", [])
             ax_tree_path = output_dir / "accessibility_tree.json"
-            with open(ax_tree_path, "w", encoding="utf-8") as f:
-                json.dump({
-                    "success": True,
-                    "timestamp": datetime.now().isoformat(),
-                    "url": driver.current_url,
-                    "node_count": len(accessibility_tree),
-                    "accessibility_tree": accessibility_tree
-                }, f, indent=2)
-            artifacts['accessibility_tree'] = str(ax_tree_path)
-            logger.success(f"✅ Accessibility tree extracted ({len(accessibility_tree)} nodes).")
-        except Exception as ax_error:
-            logger.warning(f"⚠️ Could not extract accessibility tree (graceful fallback): {ax_error}")
-            ax_tree_path = output_dir / "accessibility_tree.json"
-            with open(ax_tree_path, "w", encoding="utf-8") as f:
-                json.dump({ "success": False, "error": str(ax_error) }, f, indent=2)
+            ax_tree_path.write_text(json.dumps({"success": True, "node_count": len(ax_tree), "accessibility_tree": ax_tree}, indent=2), encoding='utf-8')
             artifacts['accessibility_tree'] = str(ax_tree_path)
 
-        logger.info("📄 Summarizing accessibility tree...")
-        summary_result = await dom_tools.summarize_accessibility_tree({"file_path": str(ax_tree_path)})
-        if summary_result.get("success"):
-            summary_path = output_dir / "accessibility_tree_summary.txt"
-            summary_path.write_text(summary_result["output"], encoding='utf-8')
-            artifacts['accessibility_tree_summary'] = str(summary_path)
-            logger.success("✅ Accessibility tree summary saved.")
-        else:
-            logger.warning(f"⚠️ Could not summarize accessibility tree: {summary_result.get('error')}")
+            summary_result = await dom_tools.summarize_accessibility_tree({"file_path": str(ax_tree_path)})
+            if summary_result.get("success"):
+                summary_path = output_dir / "accessibility_tree_summary.txt"
+                summary_path.write_text(summary_result["output"], encoding='utf-8')
+                artifacts['accessibility_tree_summary'] = str(summary_path)
+        except Exception as ax_error:
+            logger.warning(f"⚠️ Could not extract accessibility tree: {ax_error}")
 
         logger.success(f"✅ Scrape successful for {url}")
         return {"success": True, "looking_at_files": artifacts}
@@ -215,4 +162,7 @@ async def selenium_automation(params: dict) -> dict:
     finally:
         if driver:
             driver.quit()
-            logger.info("Browser closed.")
+            if verbose: logger.info("Browser closed.")
+        if temp_profile and profile_path and os.path.exists(profile_path):
+             shutil.rmtree(profile_path)
+             if verbose: logger.info(f"Cleaned up temporary profile: {profile_path}")

[mike@nixos:~/repos/pipulate]$