AI Stress Test: o1 Pro

class LinkFlow:
    """
    Example “Pipulate-style” multi-step workflow for Botify link graphs.

    Steps:
      1) Botify token check (step_01)
      2) Org/Project selection (step_02)
      3) Analysis selection (step_03)
      4) Export start (step_04)
      5) Final or poll (step_05)

    Each step is stored in a single pipeline record keyed by the pipeline_id (e.g. "linkflow").
    We use Pipulate’s .set_step_data(url, step_id, {data}) and .get_step_data(url, step_id).
    """

    def __init__(self, app, pipulate, route_prefix="/linkflow"):
        self.app = app
        self.pipulate = pipulate
        self.prefix = route_prefix

        # Define your steps as (key, step_id, label)
        self.STEPS = [
            ("token",   "step_01", "Botify Token"),
            ("project", "step_02", "Pick Org & Project"),
            ("analysis","step_03", "Pick Analysis"),
            ("export",  "step_04", "Export Links & Meta"),
            ("done",    "step_05", "Finalize"),
        ]

        # Register routes (similar to how poetflow does)
        routes = [
            (f"{self.prefix}/init",       self.init,       ["POST"]),
            (f"{self.prefix}/step_01",    self.step_01),
            (f"{self.prefix}/step_01_set",self.step_01_set, ["POST"]),
            (f"{self.prefix}/step_02",    self.step_02),
            (f"{self.prefix}/step_02_set",self.step_02_set, ["POST"]),
            (f"{self.prefix}/step_03",    self.step_03),
            (f"{self.prefix}/step_03_set",self.step_03_set, ["POST"]),
            (f"{self.prefix}/step_04",    self.step_04),
            (f"{self.prefix}/export_start", self.export_start, ["POST"]),
            (f"{self.prefix}/step_05",    self.step_05),
            (f"{self.prefix}/poll_links", self.poll_links,   ["GET"]),
            (f"{self.prefix}/poll_meta",  self.poll_meta,    ["GET"]),
        ]
        for path, handler, *methods in routes:
            method_list = methods[0] if methods else ["GET"]
            app.route(path, methods=method_list)(handler)

    async def start_form(self):
        """
        The initial form that starts the link-flow. Think of it like poetflow's start_form.
        """
        return Card(
            H2("Botify Link Graph Flow"),
            Form(
                # Example: prompt user for a pipeline_id (just a text input)
                Input(name="pipeline_id", placeholder="Enter a unique label, e.g. 'linkflow'"),
                Button("Begin Workflow", type="submit"),
                hx_post=f"{self.prefix}/init",
                hx_target="#linkflow-container"
            ),
            id="linkflow-init-form"
        )

    async def init(self, request):
        """
        Initialize the pipeline, storing the pipeline_id user typed in. 
        Then return placeholders for all steps (similar to poetflow).
        """
        form = await request.form()
        pipeline_id = form.get("pipeline_id", "linkflow").strip()
        db["pipeline_id"] = pipeline_id  # store in your DictLikeDB

        # Initialize if missing in pipulate
        self.pipulate.initialize_if_missing(pipeline_id)

        # For example, ensure step_01 is set if not present
        if not self.pipulate.get_step_data(pipeline_id, "step_01"):
            self.pipulate.set_step_data(pipeline_id, "step_01", {"token_verified": False})

        return Div(
            H3("Link Graph Workflow Started"),
            # placeholders for each step
            Div(id="step_01", hx_get=f"{self.prefix}/step_01", hx_trigger="load"),
            Div(id="step_02"),
            Div(id="step_03"),
            Div(id="step_04"),
            Div(id="step_05"),
            id="linkflow-container"
        )

    async def step_01(self, request):
        """
        Step 01: Botify token check. 
        If token_verified is True, skip to step_02. Otherwise, prompt user for token.
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        step1_data = self.pipulate.get_step_data(pipeline_id, "step_01", {})
        if step1_data.get("token_verified"):
            # Already verified, load step_02
            return Div(
                Card(f"Step 1: Botify token is verified ✓"),
                Div(id="step_02", hx_get=f"{self.prefix}/step_02", hx_trigger="load")
            )
        else:
            # Not verified, show form to enter token
            return Div(
                Card(
                    H3("Step 1: Enter Botify Token"),
                    Form(
                        Input(type="text", name="botify_token", placeholder="Paste token here"),
                        Button("Verify", type="submit"),
                        hx_post=f"{self.prefix}/step_01_set",
                        hx_target="#step_01"
                    ),
                ),
                Div(id="step_02"),  # placeholder, to be loaded next
                id="step_01"
            )

    async def step_01_set(self, request):
        """
        Handle token submission and attempt verification. If successful, set token_verified = True
        """
        form = await request.form()
        token = form.get("botify_token", "")
        pipeline_id = db.get('pipeline_id', 'linkflow')

        # 1) Verify token (you can reuse your old verify_botify_token logic)
        verified = await self._verify_token(token)  # a hypothetical helper

        if verified:
            self.pipulate.set_step_data(pipeline_id, "step_01", {"token_verified": True, "token": token})
            await chatq("Botify token verified, moving to Step 2.")
            return Div(
                Card("Step 1: Token verified ✓"),
                Div(id="step_02", hx_get=f"{self.prefix}/step_02", hx_trigger="load")
            )
        else:
            await chatq("Botify token verification failed.")
            return Card("❌ Token verification failed. Please try again.", style="color:red;")

    async def _verify_token(self, token) -> bool:
        """
        Minimal placeholder for real token verification logic.
        Return True if valid, else False.
        """
        # e.g., check with Botify API
        return bool(token.strip())  # simplistic stub

    async def step_02(self, request):
        """
        Step 2: Org & Project selection. 
        If chosen, skip to step_03, else show a form.
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        step2_data = self.pipulate.get_step_data(pipeline_id, "step_02", {})
        if "org" in step2_data and "project" in step2_data:
            return Div(
                Card(f"Step 2: Org/Project chosen: {step2_data['org']}/{step2_data['project']} ✓"),
                Div(id="step_03", hx_get=f"{self.prefix}/step_03", hx_trigger="load")
            )
        else:
            # Show form for picking org/project 
            org_projects = ["org1/project1", "org2/project2"]  # or parse from candidates.txt
            return Div(
                Card(
                    H3("Step 2: Pick Org & Project"),
                    Form(
                        Select(
                            *[Option(path, value=path) for path in org_projects],
                            name="org_project"
                        ),
                        Button("Select", type="submit"),
                        hx_post=f"{self.prefix}/step_02_set",
                        hx_target="#step_02"
                    )
                ),
                Div(id="step_03"),
                id="step_02"
            )

    async def step_02_set(self, request):
        """
        Handle chosen org/project. 
        Store in pipeline, move on to step_03.
        """
        form = await request.form()
        selection = form.get("org_project", "")
        pipeline_id = db.get('pipeline_id', 'linkflow')

        if '/' in selection:
            org, project = selection.split('/', 1)
            self.pipulate.set_step_data(pipeline_id, "step_02", {"org": org, "project": project})
            return Div(
                Card(f"Step 2: Stored org={org}, project={project} ✓"),
                Div(id="step_03", hx_get=f"{self.prefix}/step_03", hx_trigger="load")
            )
        else:
            return P("Invalid selection. Must contain 'org/project'")

    async def step_03(self, request):
        """
        Step 3: Analysis selection. If done, load step_04; else show a form.
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        step3_data = self.pipulate.get_step_data(pipeline_id, "step_03", {})
        if "analysis" in step3_data:
            return Div(
                Card(f"Step 3: Analysis {step3_data['analysis']} chosen ✓"),
                Div(id="step_04", hx_get=f"{self.prefix}/step_04", hx_trigger="load")
            )
        else:
            # Show a form with analysis options
            analyses = ["20231015", "20230930", "20230901"]  # example
            return Div(
                Card(
                    H3("Step 3: Pick Analysis"),
                    Form(
                        Select(
                            *[Option(ana, value=ana) for ana in analyses],
                            name="analysis"
                        ),
                        Button("Choose", type="submit"),
                        hx_post=f"{self.prefix}/step_03_set",
                        hx_target="#step_03"
                    )
                ),
                Div(id="step_04"),
                id="step_03"
            )

    async def step_03_set(self, request):
        form = await request.form()
        analysis = form.get("analysis", "")
        pipeline_id = db.get('pipeline_id', 'linkflow')
        self.pipulate.set_step_data(pipeline_id, "step_03", {"analysis": analysis})
        return Div(
            Card(f"Step 3: You chose analysis: {analysis} ✓"),
            Div(id="step_04", hx_get=f"{self.prefix}/step_04", hx_trigger="load")
        )

    async def step_04(self, request):
        """
        Step 4: Kick off link export & meta export. 
        If they’re completed or in progress, we can skip to poll. 
        Otherwise, show a button to start the process.
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        step4_data = self.pipulate.get_step_data(pipeline_id, "step_04", {})
        if "export_started" in step4_data:
            return Div(
                Card("Step 4: Export In Progress, Polling..."),
                Div(id="step_05", hx_get=f"{self.prefix}/step_05", hx_trigger="load")
            )
        else:
            return Card(
                H3("Step 4: Start Exports"),
                P("Click the button below to start links + meta export"),
                Form(
                    Button("Begin Export", type="submit"),
                    hx_post=f"{self.prefix}/export_start",
                    hx_target="#step_04"
                ),
                id="step_04"
            )

    async def export_start(self, request):
        """
        Actually start the link export via your old API logic, then store export_started = True.
        Then return step_05 placeholder for polling or finalizing.
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        self.pipulate.set_step_data(pipeline_id, "step_04", {"export_started": True})
        await chatq("Link export job started! Polling soon.")
        return Div(
            Card("Step 4: Exports started ✓"),
            Div(id="step_05", hx_get=f"{self.prefix}/step_05", hx_trigger="load")
        )

    async def step_05(self, request):
        """
        Step 5: Check final status or poll. 
        If done, we show final result. If not done, poll again with hx-get.
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        final_data = self.pipulate.get_step_data(pipeline_id, "step_05", {})
        if final_data.get("done"):
            return Card(
                H2("Workflow Complete"),
                P("Link + Meta exports are done!"),
                style="color: green;"
            )
        else:
            # Show a card with poll links
            # Or just poll automatically
            return Div(
                Card("Step 5: Polling export status..."),
                # auto-poll end points
                Div(id="poll-links",  hx_get=f"{self.prefix}/poll_links", hx_trigger="load delay:3s"),
                Div(id="poll-meta",   hx_get=f"{self.prefix}/poll_meta",  hx_trigger="load delay:3s"),
                id="step_05"
            )

    async def poll_links(self, request):
        """
        Check link export status from Botify. If done, set step_05->links_done = True 
        or set step_05->done = True if meta also done, then re-render. 
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        # ... your logic ...
        # Suppose we got "done"
        done = True  # in real code, check the job
        step5_data = self.pipulate.get_step_data(pipeline_id, "step_05", {})
        step5_data["links_done"] = True
        if step5_data.get("meta_done"):
            step5_data["done"] = True
        self.pipulate.set_step_data(pipeline_id, "step_05", step5_data)

        if step5_data.get("done"):
            return Div("Links done. Everything done!", hx_get=f"{self.prefix}/step_05", hx_trigger="load")
        return P("Links export: in progress...")

    async def poll_meta(self, request):
        """
        Same logic as poll_links, but for meta. 
        """
        pipeline_id = db.get('pipeline_id', 'linkflow')
        done = True  # real logic
        step5_data = self.pipulate.get_step_data(pipeline_id, "step_05", {})
        step5_data["meta_done"] = True
        if step5_data.get("links_done"):
            step5_data["done"] = True
        self.pipulate.set_step_data(pipeline_id, "step_05", step5_data)

        if step5_data.get("done"):
            return Div("Meta done. Everything done!", hx_get=f"{self.prefix}/step_05", hx_trigger="load")
        return P("Meta export: in progress...")

# Configure app by unpacking the returned glboal scope (table, Dataclass) tuple pairs (singular, Plural)
app, rt, (store, Store), (tasks, Task), (clients, Client), (pipeline, Pipeline) = fast_app(
    "data/data.db",
    ws_hdr=True,  # Turns on WebSockets for 2-way chat
    live=True,    # Make edit, check page, make edit, check page... this is how.
    default_hdrs=False,  # See all that hdrs stuff immediately below I want to control deliberately? Needs this.
    hdrs=(
        Meta(charset='utf-8'),              # Best to let your browser know your encoding sooner rather than later
        Link(rel='stylesheet', href='/static/pico.min.css'),  # We load our dependencies statically around here
        Script(src='/static/htmx.min.js'),  # htmx is the backbone of the UI
        Script(src='/static/fasthtml.js'),  # FastHTML is not FastAPI. I can't emphasize this enough.
        Script(src='/static/surreal.js'),   # Enables dynamic updates to the user interface without requiring full page reloads. How to describe it? It's just...
        Script(src='/static/script.js'),    # A not-so-descriptive name for a file that cleverly scopes styles and keeps your CSS drama-free!
        Script(src='/static/Sortable.js'),  # Got a UL with LI's and want to make them drag-and-drop sortable? This is how.
        create_chat_scripts('.sortable'),   # All the early pageload JavaScript not part of above.
        Script(type='module')               # Because FastHTML has a bug and I need to include this to force the correct JS import pattern.
    ),
    store={            # server-side DictLikeDB store used for persistence
        "key": str,    # Key is the primary key
        "value": str,  # Value is the value of the key
        "pk": "key"    # Never twice the same key (updates override)
    },
    task={                  # Exposed to user as "task" endpoint but hardwired to "todo" in the wiring. New instances will have to accomodate in their render_item() method.
        "id": int,          # We lean into the strengths of SQLite. Auto-increment primary key work well.
        "name": str,        # Changed from "title" to "name"
        "done": bool,       # Done is a boolean flag to indicate if the task is completed
        "priority": int,    # Integrates beautifully with Sortable.js to sort tasks by priority
        "profile_id": int,  # Foreign key to profile for use with MiniDataAPI Spec .xtra() extract filter to filter TodoApp by profile
        "pk": "id"          # A task by any other name is still a todo item or generic linked-list CRUD app
    },
    client={               # "client" exposed to user as endpoint but hardwired to "profile" in the wiring of plugin element IDs in Web UI
        "id": int,         # To be defined as a SQLite auto-increment primary key via MiniDataAPI Spec
        "name": str,       # Name is actually hidden on the menu so real client names are never exposed unless in client (profile) list app
        "menu_name": str,  # Menu name is exposed on the menu so user can switch profiles in front of client without showing other client names
        "address": str,    # Address is actually used for website domain to control other apps like gap analysis
        "code": str,       # Code is actually country code used to control data-pull filters in API integrations like SEMRush
        "active": bool,    # Active lets you toggle the profile on and off in the menu
        "priority": int,   # Controls the sort order of the profile in the menu
        "pk": "id"         # Default SQLite auto-increment primary key so name and menu_name can be freely changed
    },
    pipeline={           # To "pipulate" is use this for a Unix pipe-like "pipeline" workflow: Card 1 | Card 2 | Card 3
        "url": str,      # A url must be used on Card 1 to initiate a job, and can be plugged in later to from last complete Card step
        "data": str,     # All jobs get just 1 pipulate record and use a JSON blob to track state for the entire workflow. The JSON blog contains the args and results of each Card for interruptionless pattern
        "created": str,  # ISO timestamp of first insert
        "updated": str,  # ISO timestamp of last update
        "pk": "url"      # URL is the primary key and can always be plugged into Card 1 to continue a job, jumping to where it left off (the same behavior as any step of workflow processing)
    }                    # A FastHTML-friendly querystring-like path can be used to jump to any Card in the workflow: /endpoint/card3
)

class Pipulate:
    """
    Pipulate manages a pipeline using a JSON blob with keys like "step_01", "step_02", etc.
    No 'steps' or 'current_step' keys exist. The presence of step keys determines progress.

    Data Structure Example:
    {
        "step_01": {"name": "John"},
        "step_02": {"color": "blue"},
        "created": "2024-12-08T12:34:56", 
        "updated": "2024-12-08T12:35:45"
    }

    The highest step number that exists in the JSON is considered the last completed step.
    The next step is always one more than the highest completed step.
    """

    def __init__(self, table):
        self.table = table

    def _get_state(self, url: str) -> dict:
        record = self.table[url]
        state = json.loads(record.data)
        return state

    def _save_state(self, url: str, state: dict):
        now = datetime.now().isoformat()
        state["updated"] = now
        self.table.update({
            "url": url,
            "data": json.dumps(state),
            "updated": state["updated"]
        })

    def initialize_if_missing(self, url: str, initial_step_data: dict = None) -> dict:
        """Initialize state for url if it doesn't exist"""
        try:
            return self._get_state(url)
        except NotFoundError:
            now = datetime.now().isoformat()
            state = {
                "created": now,
                "updated": now
            }
            if initial_step_data:
                state.update(initial_step_data)
            self.table.insert({
                "url": url,
                "data": json.dumps(state),
                "created": now,
                "updated": now
            })
            return state

    def get_state(self, url: str) -> dict:
        """Get current state for url"""
        try:
            return self._get_state(url)
        except NotFoundError:
            return {}

    def set_step_data(self, url: str, step_name: str, data: dict):
        """Set data for a specific step"""
        state = self.get_state(url)
        state[step_name] = data
        self._save_state(url, state)

    def get_all_step_data(self, url: str, steps, exclude_final=True) -> dict:
        """Gather data from all completed steps.
        
        Args:
            url: Workflow identifier
            steps: List of workflow steps
            exclude_final: Whether to exclude the final step
        """
        step_range = steps[:-1] if exclude_final else steps
        step_data = {}
        for key, step_id, _ in step_range:
            data = self.get_step_data(url, step_id, {})
            step_data[key] = data.get(key, "???")
        return step_data

    @pipeline_operation
    def get_step_data(self, url: str, step_name: str, default=None) -> dict:
        """Get data for a specific step"""
        state = self.get_state(url)
        return state.get(step_name, default or {})

    def get_last_completed_step_number(self, url: str, steps) -> int:
        """Get highest completed step number from defined workflow steps."""
        state = self.get_state(url)
        
        # Work backwards through steps to find last completed one
        for i, (_, step_id, _) in reversed(list(enumerate(steps))):
            if step_id in state:
                return i + 1
        return 0

    @pipeline_operation
    def should_advance(self, url: str, current_step: str, condition: dict) -> bool:
        """Check if step should advance based on condition

        Example:
        if pipulate.should_advance(url, "step_02", {"color": "*"}):
            # Move to step 3
        """
        step_data = self.get_step_data(url, current_step)
        return all(k in step_data for k in condition.keys())

    def generate_step_placeholders(self, steps, prefix, start_from=0):
        """Generate step placeholder divs for any workflow.
        
        Args:
            steps: List of (key, step_id, label) tuples defining the workflow
            prefix: URL prefix for the workflow (e.g., "/poetx")
            start_from: Index of step to trigger on load (default 0)
            
        Returns:
            List of Div elements with appropriate HTMX attributes
        """
        return [
            Div(
                id=step_id,
                hx_get=f"{prefix}/{step_id}",
                hx_trigger="load" if i == start_from else None
            )
            for i, (_, step_id, _) in enumerate(steps)
        ]

    def clear_steps_from(self, url: str, target_step: str, steps):
        """Clear state from target step onwards.
        
        Args:
            url: Workflow identifier
            target_step: Step ID to start clearing from
            steps: List of workflow steps
        Returns:
            Updated state dict
        """
        state = self.get_state(url)
        step_indices = {step_id: i for i, (_, step_id, _) in enumerate(steps)}
        target_idx = step_indices[target_step]
        
        for _, step_id, _ in steps[target_idx:]:
            state.pop(step_id, None)
        
        self._save_state(url, state)
        return state

    def generate_step_chain(self, prefix: str, url: str, steps) -> Div:
        """Build chain of step placeholders up to next incomplete step.
        
        Args:
            prefix: URL prefix for the workflow
            url: Workflow identifier
            steps: List of workflow steps
        """
        last_step = self.get_last_completed_step_number(url)
        next_step = last_step + 1
        
        placeholders = [
            Div(
                id=step_id,
                hx_get=f"{prefix}/{step_id}",
                hx_trigger="load" if i == 0 else None,
                hx_swap="outerHTML"
            )
            for i, (_, step_id, _) in enumerate(steps[:next_step])
        ]
        
        return Div(*placeholders)

    def get_step_summary(self, url: str, current_step: str, steps) -> tuple[dict, list]:
        """Get state and summary up to current step.
        
        Args:
            url: Workflow identifier
            current_step: Current step being processed
            steps: List of workflow steps
            
        Returns:
            (state_dict, summary_lines) where:
            - state_dict: {key: value} of completed steps
            - summary_lines: List of formatted "Label: value" strings
        """
        # Get state up to current step
        state = {}
        current_step_found = False
        for key, step_id, label in steps:
            if current_step_found:
                break
            if step_id == current_step:
                current_step_found = True
            step_data = self.get_step_data(url, step_id, {})
            if key in step_data:
                state[key] = step_data[key]

        # Build summary lines
        summary_lines = []
        for key, step_id, label in steps:
            if step_id == current_step:
                break
            if key in state:
                summary_lines.append(f"- {label}: {state[key]}")

        return state, summary_lines


# Global instance - module scope is the right scope
pipulate = Pipulate(pipeline)

class Poetflow:
    """
    Revised Poetflow:
    - 5-step workflow similar to Workflow & Petiteflow.
    - Each step returns a single Div(id="step_xx") at top-level.
    - Completed steps auto-load the next step.
    - Completed steps are clickable to return to that step.
    - Unfinalize allows reverting finalization.
    - New Poem triggers a full refresh.

    Design:
    - A stable headline card at the top.
    - A #pipeline-container below it for steps.
    - On finalize/unfinalize, we only update #pipeline-container, not the headline card.
    """

    def __init__(self, app, pipulate):
        self.app = app
        self.pipulate = pipulate
        self.prefix = "/poetx"
        self.logger = logger.bind(name="PoetflowExplained")
        self.llm_enabled = True  # Toggle for LLM output

        # Define the steps for clarity and future extensibility
        self.STEPS = [
            ("name", "step_01", "Name"),
            ("color", "step_02", "Color"), 
            ("time", "step_03", "Time"),
            ("feeling", "step_04", "Feeling"),
            ("finalized", "step_05", "Finalize")
        ]

        routes = [
            (f"{self.prefix}/init", self.init, ["POST"]),
            (f"{self.prefix}/step_01", self.step_01),
            (f"{self.prefix}/step_02", self.step_02),
            (f"{self.prefix}/step_02_submit", self.step_02_submit, ["POST"]),
            (f"{self.prefix}/step_03", self.step_03),
            (f"{self.prefix}/step_03_submit", self.step_03_submit, ["POST"]),
            (f"{self.prefix}/step_04", self.step_04),
            (f"{self.prefix}/step_04_submit", self.step_04_submit, ["POST"]),
            (f"{self.prefix}/step_05", self.step_05),
            (f"{self.prefix}/step_05_submit", self.step_05_submit, ["POST"]),
            (f"{self.prefix}/unfinalize", self.unfinalize, ["POST"]),
            (f"{self.prefix}/new_poem", self.new_poem, ["POST"]),
            (f"{self.prefix}/return_to_step", self.return_to_step, ["POST"])
        ]

        for path, handler, *methods in routes:
            method_list = methods[0] if methods else ["GET"]
            self.logger.debug(f"  {path} -> {handler.__name__} ({', '.join(method_list)})")
            app.route(path, methods=method_list)(handler)

    async def start_form(self):
        return Card(
            H2("Create a Poem with Explanations"),
            P("We use the pipeline-based UI and LLM enhancements."),
            Form(
                Input(
                    name="name",
                    placeholder="Enter your name",
                    required=True,
                    value=db.get('pipeline_id', '')
                ),
                Button("Begin"),
                hx_post=f"{self.prefix}/init",
                hx_target="#poemx"
            ),
            id="poemx"
        )

    def explain(self, url, current_step, message):
        if not self.llm_enabled:
            return

        # Get step summary from Pipulate
        _, summary_lines = self.pipulate.get_step_summary(url, current_step, self.STEPS)
        
        # Format the prompt
        summary = ""
        if summary_lines:
            summary = "So far:\n" + "\n".join(summary_lines) + "\n\n"

        prompt = (
            f"Explain {limiter} what is happening at '{current_step}'.\n\n"
            f"{summary}"
            f"{message}"
        )
        asyncio.create_task(chatq(prompt, role="system"))

    async def generate_poem(self, name, color, time, feeling):
        if not self.llm_enabled:
            return "LLM output disabled. This is a placeholder poem."

        prompt = (
            f"Create a short poem {limiter} incorporating these elements:\n"
            f"- Name: {name}\n"
            f"- Color: {color}\n"
            f"- Time: {time}\n"
            f"- Feeling: {feeling}\n\n"
            f"Make it cohesive and beautiful."
        )

        poem = []
        messages = [{"role": "user", "content": prompt}]
        async for chunk in chat_with_llm(model, messages):
            poem.append(chunk)
        final_poem = "".join(poem).strip()
        return final_poem

    async def init(self, request):
        form = await request.form()
        name = form.get("name")
        db['pipeline_id'] = name

        self.pipulate.initialize_if_missing(name)
        if not self.pipulate.get_step_data(name, "step_01"):
            self.pipulate.set_step_data(name, "step_01", {"name": name})

        if self.pipulate.should_advance(name, "step_05", {"finalized": "*"}):
            final_poem = self.pipulate.get_step_data(name, "poem", {}).get("text", "No poem generated.")
            return self.render_full_view_finalized(final_poem)

        return Div(
            Card(
                H2("Create a Poem with Explanations"),
                P("We use the pipeline-based UI and LLM enhancements.")
            ),
            Div(
                *self.pipulate.generate_step_placeholders(self.STEPS, self.prefix),
                id="pipeline-container"
            )
        )

    def render_full_view_finalized(self, final_poem):
        """Render full view when already finalized at init."""
        return Div(
            Card(
                H2("Create a Poem with Explanations"),
                P("We use the pipeline-based UI and LLM enhancements.")
            ),
            Div(
                self.render_final_card_contents(final_poem),
                id="pipeline-container"
            )
        )

    def render_final_card_contents(self, final_poem):
        """Render just the inner card for the finalized poem, without headline, for pipeline-container."""
        return Card(
            H3("✨ Step 5: Poem Complete ✨"),
            P("Your masterpiece is ready!"),
            P(final_poem),
            Div(
                Form(
                    Button("Unfinalize", type="submit"),
                    hx_post=f"{self.prefix}/unfinalize",
                    hx_target="#pipeline-container",
                    hx_swap="outerHTML"
                ),
                Form(
                    Button("New Poem", type="submit"),
                    hx_post=f"{self.prefix}/new_poem",
                    hx_target="#pipeline-container"
                ),
                style="display: flex; gap: 1rem; margin-top: 1rem;"
            )
        )

    async def step_01(self, request):
        """First step shows name (and is never clickable)"""
        name = db['pipeline_id']
        step1_data = self.pipulate.get_step_data(name, "step_01")

        # Step 01 completed by init already, just show it and trigger step_02
        return Div(
            Card(f"Step 1: Hello, {step1_data['name']} ✓"),
            Div(id="step_02", hx_get=f"{self.prefix}/step_02", hx_trigger="load")
        )

    async def step_02(self, request):
        name = db['pipeline_id']
        step2_data = self.pipulate.get_step_data(name, "step_02")
        if step2_data:
            return Div(
                Card(
                    f"Step 2: Color chosen: {step2_data['color']} ✓",
                    hx_post=f"{self.prefix}/return_to_step",
                    hx_vals='{"step": "02"}',
                    hx_target="#pipeline-container",
                    hx_swap="outerHTML"
                ),
                Div(id="step_03", hx_get=f"{self.prefix}/step_03", hx_trigger="load", hx_swap="outerHTML"),
                id="step_02"
            )
        else:
            self.explain(name, "Step 02", "Pick a color for the poem.")
            return Div(
                Card(
                    H3("Step 2: Pick a color"),
                    Form(
                        Select(
                            Option("Red", value="red"),
                            Option("Blue", value="blue"),
                            Option("Green", value="green"),
                            name="color"
                        ),
                        Button("Choose"),
                        hx_post=f"{self.prefix}/step_02_submit",
                        hx_target="#step_02"
                    )
                ),
                Div(id="step_03"),
                id="step_02"
            )

    async def step_02_submit(self, request):
        form = await request.form()
        color = form.get("color")
        name = db['pipeline_id']
        self.pipulate.set_step_data(name, "step_02", {"color": color})
        self.explain(name, "Step_02_Submitted", f"Color '{color}' chosen. Next: choose time.")
        return Div(
            Card(
                f"Step 2: You chose {color} ✓",
                hx_post=f"{self.prefix}/return_to_step",
                hx_vals='{"step": "02"}',
                hx_target="#pipeline-container",
                hx_swap="outerHTML"
            ),
            Div(id="step_03", hx_get=f"{self.prefix}/step_03", hx_trigger="load", hx_swap="outerHTML"),
            id="step_02"
        )

    async def step_03(self, request):
        name = db['pipeline_id']
        step3_data = self.pipulate.get_step_data(name, "step_03")
        if step3_data:
            return Div(
                Card(
                    f"Step 3: Time chosen: {step3_data['time']} ✓",
                    hx_post=f"{self.prefix}/return_to_step",
                    hx_vals='{"step": "03"}',
                    hx_target="#pipeline-container",
                    hx_swap="outerHTML"
                ),
                Div(id="step_04", hx_get=f"{self.prefix}/step_04", hx_trigger="load", hx_swap="outerHTML"),
                id="step_03"
            )
        else:
            self.explain(name, "Step 03", "Pick a time (dawn/dusk/night).")
            return Div(
                Card(
                    H3("Step 3: Choose a time"),
                    Form(
                        Select(
                            Option("Dawn", value="dawn"),
                            Option("Dusk", value="dusk"),
                            Option("Night", value="night"),
                            name="time"
                        ),
                        Button("Choose"),
                        hx_post=f"{self.prefix}/step_03_submit",
                        hx_target="#step_03"
                    )
                ),
                Div(id="step_04"),
                id="step_03"
            )

    async def step_03_submit(self, request):
        form = await request.form()
        time = form.get("time")
        name = db['pipeline_id']
        self.pipulate.set_step_data(name, "step_03", {"time": time})
        self.explain(name, "Step_03_Submitted", f"Time '{time}' chosen. Next: choose feeling.")
        return Div(
            Card(
                f"Step 3: You chose {time} ✓",
                hx_post=f"{self.prefix}/return_to_step",
                hx_vals='{"step": "03"}',
                hx_target="#pipeline-container",
                hx_swap="outerHTML"
            ),
            Div(id="step_04", hx_get=f"{self.prefix}/step_04", hx_trigger="load", hx_swap="outerHTML"),
            id="step_03"
        )

    async def step_04(self, request):
        name = db['pipeline_id']
        step4_data = self.pipulate.get_step_data(name, "step_04")
        if step4_data:
            return Div(
                Card(
                    f"Step 4: Feeling chosen: {step4_data['feeling']} ✓",
                    hx_post=f"{self.prefix}/return_to_step",
                    hx_vals='{"step": "04"}',
                    hx_target="#pipeline-container",
                    hx_swap="outerHTML"
                ),
                Div(id="step_05", hx_get=f"{self.prefix}/step_05", hx_trigger="load", hx_swap="outerHTML"),
                id="step_04"
            )
        else:
            self.explain(name, "Step_04", "Pick a feeling (joy/peace/wonder).")
            return Div(
                Card(
                    H3("Step 4: Choose a feeling"),
                    Form(
                        Select(
                            Option("Joy", value="joy"),
                            Option("Peace", value="peace"),
                            Option("Wonder", value="wonder"),
                            name="feeling"
                        ),
                        Button("Choose"),
                        hx_post=f"{self.prefix}/step_04_submit",
                        hx_target="#step_04",
                    )
                ),
                Div(id="step_05"),
                id="step_04"
            )

    async def step_04_submit(self, request):
        form = await request.form()
        feeling = form.get("feeling")
        name = db['pipeline_id']
        self.pipulate.set_step_data(name, "step_04", {"feeling": feeling})
        self.explain(name, "Step_04_Submitted", f"Feeling '{feeling}' chosen. Next: preview poem.")
        return Div(
            Card(
                f"Step 4: You chose {feeling} ✓",
                hx_post=f"{self.prefix}/return_to_step",
                hx_vals='{"step": "04"}',
                hx_target="#pipeline-container",
                hx_swap="outerHTML"
            ),
            Div(id="step_05", hx_get=f"{self.prefix}/step_05", hx_trigger="load", hx_swap="outerHTML"),
            id="step_04"
        )

    async def step_05(self, request):
        name = db['pipeline_id']
        step5_data = self.pipulate.get_step_data(name, "step_05", {})
        
        if "finalized" in step5_data:
            poem_data = self.pipulate.get_step_data(name, "poem", {})
            final_poem = poem_data.get("text", "No poem generated.")
            return Div(
                self.render_final_card_contents(final_poem),
                id="pipeline-container"
            )

        # Get all step data except final step
        step_data = self.pipulate.get_all_step_data(name, self.STEPS)
        
        final_poem = await self.generate_poem(**step_data)
        self.pipulate.set_step_data(name, "poem", {"text": final_poem})
        self.explain(name, "Step_05", "Poem preview generated. Confirm to finalize.")
        
        return Div(
            Card(
                H3("Step 5: Your Poem Preview"),
                P(final_poem),
                Form(
                    Button("Perfect!", type="submit"),
                    hx_post=f"{self.prefix}/step_05_submit",
                    hx_target="#step_05"
                )
            ),
            id="step_05"
        )

    async def step_05_submit(self, request):
        name = db['pipeline_id']
        poem_data = self.pipulate.get_step_data(name, "poem", {})
        final_poem = poem_data.get("text", "No poem generated.")
        self.pipulate.set_step_data(name, "step_05", {"finalized": True})
        self.explain(name, "Final_Recap", "Poem finalized.")
        # Replace pipeline-container only
        return Div(
            self.render_final_card_contents(final_poem),
            id="pipeline-container"
        )

    async def unfinalize(self, request):
        name = db.get('pipeline_id')
        if not name:
            return P("No pipeline found", style="color:red;")

        state = self.pipulate.get_state(name)
        if "step_05" in state and "finalized" in state["step_05"]:
            del state["step_05"]["finalized"]
            self.pipulate._save_state(name, state)
            return Div(
                *self.pipulate.generate_step_placeholders(self.STEPS, self.prefix),
                id="pipeline-container"
            )
        else:
            return P("Pipeline not finalized. No changes made.")

    async def new_poem(self, request):
        """Respond with HX-Refresh to reload the page and start fresh."""
        response = HTMLResponse("")
        response.headers["HX-Refresh"] = "true"
        return response

    async def return_to_step(self, request):
        form = await request.form()
        step_num = form.get("step")
        name = db['pipeline_id']
        
        # Clear this step and all following steps
        self.pipulate.clear_steps_from(name, f"step_{step_num}", self.STEPS)
        
        return Div(
            *self.pipulate.generate_step_placeholders(self.STEPS, self.prefix),
            id="pipeline-container"
        )

class BridgeFlow:
    """
    A miniature 3-step flow paying homage to Monty Python’s Bridge of Death.
    Step 1: Name
    Step 2: Quest
    Step 3: Favorite Color
    Then either pass or fail based on whether you pick "blue" as your color.

    This is a minimal example of using Pipulate to track step data, with each step
    returning a Div containing either a form or a summary, until all steps are done.
    """

    def __init__(self, app, pipulate, prefix="/bridge"):
        self.app = app
        self.pipulate = pipulate
        self.prefix = prefix

        # Define minimal steps
        # (internal_key, step_id, display_label)
        self.STEPS = [
            ("name",   "step_01", "Name"),
            ("quest",  "step_02", "Quest"),
            ("color",  "step_03", "Color"),
        ]

        routes = [
            (f"{prefix}",               self.landing),
            (f"{prefix}/init",          self.init,         ["POST"]),
            (f"{prefix}/step_01",       self.step_01),
            (f"{prefix}/step_01_submit",self.step_01_submit,["POST"]),
            (f"{prefix}/step_02",       self.step_02),
            (f"{prefix}/step_02_submit",self.step_02_submit,["POST"]),
            (f"{prefix}/step_03",       self.step_03),
            (f"{prefix}/step_03_submit",self.step_03_submit,["POST"]),
        ]
        for path, handler, *methods in routes:
            method_list = methods[0] if methods else ["GET"]
            self.app.route(path, methods=method_list)(handler)

    async def landing(self, request):
        """
        GET /bridge
        Show a quick landing page with a single form that calls /bridge/init to begin the pipeline.
        """
        return Container(
            Card(
                H2("The Bridge of Death"),
                P("Stop! Who would cross the Bridge of Death must answer me these questions three!"),
                Form(
                    Input(
                        name="pipeline_id",
                        placeholder="Enter your name (and begin...)",
                        required=True
                    ),
                    Button("Begin"),
                    hx_post=f"{self.prefix}/init",
                    hx_target="#bridge-container"
                )
            ),
            Div(id="bridge-container")  # Where steps will be loaded
        )

    async def init(self, request):
        """
        POST /bridge/init
        Sets up the pipeline if missing and auto-loads step_01.
        """
        form = await request.form()
        pipeline_id = form.get("pipeline_id", "unknown")
        db["pipeline_id"] = pipeline_id  # Store in your server-side store if needed

        # Initialize pipeline record
        self.pipulate.initialize_if_missing(pipeline_id)

        # Return placeholders for each step, but only load step_01 first
        placeholders = self.pipulate.generate_step_placeholders(self.STEPS, self.prefix, start_from=0)
        return Div(
            *placeholders,
            id="bridge-container"
        )

    async def step_01(self, request):
        """
        GET /bridge/step_01
        Show the 'What... is your name?' question.
        """
        pipeline_id = db.get("pipeline_id", "unknown")
        step1_data = self.pipulate.get_step_data(pipeline_id, "step_01", {})

        if step1_data.get("name"):
            # Already answered, so just show summary + load next
            return Div(
                Card(f"You said your name is: {step1_data['name']} (locked) ✓"),
                Div(id="step_02", hx_get=f"{self.prefix}/step_02", hx_trigger="load")
            )
        else:
            # Show the form to ask: Name?
            return Div(
                Card(
                    H3("Question 1: What... is your name?"),
                    Form(
                        Input(type="text", name="name", placeholder="Sir Lancelot", required=True),
                        Button("Next", type="submit"),
                        hx_post=f"{self.prefix}/step_01_submit",
                        hx_target="#step_01",
                    )
                ),
                Div(id="step_02"),  # placeholder for next
                id="step_01"
            )

    async def step_01_submit(self, request):
        """
        POST /bridge/step_01_submit
        Save step_01 data and load step_02
        """
        form = await request.form()
        name = form.get("name", "")
        pipeline_id = db.get("pipeline_id", "unknown")
        self.pipulate.set_step_data(pipeline_id, "step_01", {"name": name})

        # Return summary + next step placeholder
        return Div(
            Card(f"Your name: {name} ✓ (locked)"),
            Div(id="step_02", hx_get=f"{self.prefix}/step_02", hx_trigger="load")
        )

    async def step_02(self, request):
        """
        GET /bridge/step_02
        Show the 'What... is your quest?' question.
        """
        pipeline_id = db.get("pipeline_id", "unknown")
        step2_data = self.pipulate.get_step_data(pipeline_id, "step_02", {})

        if step2_data.get("quest"):
            return Div(
                Card(f"You said your quest is: {step2_data['quest']} (locked) ✓"),
                Div(id="step_03", hx_get=f"{self.prefix}/step_03", hx_trigger="load")
            )
        else:
            return Div(
                Card(
                    H3("Question 2: What... is your quest?"),
                    Form(
                        Input(type="text", name="quest", placeholder="I seek the Grail", required=True),
                        Button("Next", type="submit"),
                        hx_post=f"{self.prefix}/step_02_submit",
                        hx_target="#step_02",
                    )
                ),
                Div(id="step_03"),
                id="step_02"
            )

    async def step_02_submit(self, request):
        """
        POST /bridge/step_02_submit
        Save step_02 data and proceed
        """
        form = await request.form()
        quest = form.get("quest", "")
        pipeline_id = db.get("pipeline_id", "unknown")
        self.pipulate.set_step_data(pipeline_id, "step_02", {"quest": quest})

        return Div(
            Card(f"Your quest: {quest} ✓ (locked)"),
            Div(id="step_03", hx_get=f"{self.prefix}/step_03", hx_trigger="load")
        )

    async def step_03(self, request):
        """
        GET /bridge/step_03
        Show the final question: 'What... is your favorite color?'
        """
        pipeline_id = db.get("pipeline_id", "unknown")
        step3_data = self.pipulate.get_step_data(pipeline_id, "step_03", {})

        if step3_data.get("color"):
            # Already answered, we can check final result
            return self._final_card(step3_data["color"])
        else:
            return Div(
                Card(
                    H3("Question 3: What... is your favorite color?"),
                    Form(
                        Select(
                            Option("Red", value="red"),
                            Option("Blue", value="blue"),
                            Option("Green", value="green")
                        , name="color"),
                        Button("Submit", type="submit"),
                        hx_post=f"{self.prefix}/step_03_submit",
                        hx_target="#step_03"
                    )
                ),
                id="step_03"
            )

    async def step_03_submit(self, request):
        """
        POST /bridge/step_03_submit
        Evaluate the color. If it's not blue, you plummet to your doom. If it's blue, you pass.
        """
        form = await request.form()
        color = form.get("color", "").lower()
        pipeline_id = db.get("pipeline_id", "unknown")
        self.pipulate.set_step_data(pipeline_id, "step_03", {"color": color})

        return self._final_card(color)

    def _final_card(self, color):
        """
        Present the final pass/fail card based on the color chosen.
        """
        if color == "blue":
            return Card(
                H3("Bridge Keeper: Right. Off you go."),
                P("You have chosen wisely. You may pass the Bridge of Death safely!"),
                style="color: green;",
                id="bridge-final"
            )
        else:
            return Card(
                H3("Bridge Keeper: ARRRGH!!!"),
                P(f"'{color.capitalize()}'?! That is the WRONG answer. You are cast into the Gorge of Eternal Peril!"),
                style="color: red;",
                id="bridge-final"
            )

    async def landing(self):

    if menux == profile_app.name:
        return await profile_render()  # Might look like a plugin, but integral.
    elif menux == todo_app.name:
        return await todo_render(menux, render_items)
    elif menux == 'connect_with_botify':
        return await connect_with_botify.connect_render()
    elif menux == 'link_graph':
        return await link_graph.link_render()
    elif menux == 'stream_simulator':
        return await stream_simulator.stream_render()
    elif menux == 'workflow':
        return await workflow.start_form()
    elif menux == 'bridgeflow':
        return await bridgeflow.landing()
    elif menux == 'poetflow':
        return await poetflow.start_form()
    else:
        return await introduction.introduction_render()

class ConnectWithBotify:
    def __init__(self, app, route_prefix="/connect-to-botify", id_suffix=""):
        self.app = app
        self.route_prefix = route_prefix
        self.id_suffix = id_suffix
        self.logger = logger.bind(name="BotifyLinkGraph").bind(suffix=id_suffix)

        # API URL templates
        self.base_url = "https://api.botify.com/v1"
        self.auth_url = f"{self.base_url}/authentication/profile"
        self.token_file = Path('botify_token.txt')  # Look in repo root
        self.current_project_url = None  # Store current project URL
        self.api_key = api_key
        self.username = username

        self.app.add_route(f"{route_prefix}/verify_token", self.verify_botify_token, methods=['POST'])
        self.logger.debug("Initialized BotifyLinkGraph with prefix: {}", route_prefix)

    async def verify_botify_token(self, request):
        """Handle Botify token verification."""
        global username, api_key
        form = await request.form()
        api_key = form.get('botify_token')

        # Verify token with API
        headers = {"Authorization": f"Token {api_key}"}
        async with aiohttp.ClientSession() as session:
            try:
                async with session.get(self.auth_url, headers=headers) as response:
                    if response.status == 200:
                        profile = await response.json()
                        username = profile["data"]["username"]
                        self.token_file.write_text(api_key)
                        self.api_key = api_key
                        self.username = username
                        global endpoint_messages
                        endpoint_messages = build_endpoint_messages(api_key=api_key, username=username)
                        asyncio.create_task(
                            chatq(f"Tell the user {limiter} Botify integration verified for {username}.")
                        )
                        return self.create_integration_card()
                    else:
                        asyncio.create_task(
                            chatq(f"Tell the user {limiter} Failed to verify the Botify token. Please try again.")
                        )
                        return self.create_integration_card(error_message="Failed to verify the Botify token. Please try again.")
            except Exception as e:
                self.logger.error(f"Error getting user profile: {str(e)}")
                return self.create_integration_card(error_message="Failed to verify the Botify token. Please try again.")

    def create_integration_card(self, error_message=None):
        """Create the Botify integration card."""
        if self.username:
            return Article(
                Header(H2("Botify API Connected")),
                P(f"Welcome, {self.username.title().replace('.', ' ')}! The Botify API is active."),
                P("You can now use the Link Graph features."),
                id=f"botify-integration-card{self.id_suffix}"
            )
        else:
            return Card(
                Header(H2("Botify Integration")),
                P("Botify integration is not set up. Please add your Botify token to use the Link Graph features."),
                P("Botify Employee? ", A("Find your token here", href="https://app.botify.com/account/", target="_blank")),
                P(error_message, style="color: red;") if error_message else None,
                Form(
                    Div(
                        Input(
                            type="text",
                            name="botify_token",
                            placeholder="Paste your Botify API token here",
                            required=True,
                            style="flex-grow: 1; margin-right: 10px;"
                        ),
                        Button("Verify Token", type="submit"),
                        style="display: flex; align-items: center; margin-bottom: 10px;"
                    ),
                    hx_post=f"{self.route_prefix}/verify_token",
                    hx_target=f"#botify-integration-card{self.id_suffix}",
                    hx_swap="outerHTML"
                ),
                id=f"botify-integration-card{self.id_suffix}",
            )

    async def connect_render(self):
        """Render the main link graph content."""
        main_card = self.create_integration_card()
        return Div(
            main_card,
            Div(id="update-status"),
            id=f"content-container{self.id_suffix}"
        )


connect_with_botify = ConnectWithBotify(app, route_prefix="/connect-to-botify", id_suffix="")


class BotifyLinkGraph:
    """A FastHTML component for interacting with the Botify API to generate link graphs.

    This class handles the UI and API interactions for:
    - Confirms Botify API connection
    - Project/analysis selection
    - Link graph data export
    - Progress monitoring

    Flow:
    1. Initialization (__init__):
       - Sets up API endpoints and state
       - Creates download directory
       - Registers FastHTML routes for all API interactions

    2. Initial Render (render):
       - Called when user visits the root route (/link-graph/)
       - Checks for Botify token and user profile
       - If authenticated:
           - Shows welcome card with username
           - Displays org/project selection form
       - If not authenticated:
           - Shows token input form

    3. Subsequent Interactions:
       - Token verification -> verify_botify_token
       - Project updates -> update_project
       - Analysis processing -> process_analysis
    """

    def __init__(self, app, route_prefix="/link-graph", id_suffix=""):
        self.app = app
        self.route_prefix = route_prefix
        self.id_suffix = id_suffix
        self.logger = logger.bind(name="BotifyLinkGraph").bind(suffix=id_suffix)

        # API URL templates
        self.base_url = "https://api.botify.com/v1"
        self.auth_url = f"{self.base_url}/authentication/profile"
        self.export_params = None  # Will be set during export creation
        self.selected_fields = [
            'url',
            'crawl.outlinks_internal.graph.url'
        ]

        self.token_file = Path('botify_token.txt')  # Look in repo root
        self.current_project_url = None  # Store current project URL

        # Create downloads directory matching endpoint name
        self.downloads_dir = Path("downloads") / route_prefix.lstrip("/")
        self.downloads_dir.mkdir(parents=True, exist_ok=True)
        self.graphs = self.downloads_dir / "graphs"
        self.graphs.mkdir(parents=True, exist_ok=True)
        self.logger.debug(f"Initialized downloads directory at: {self.downloads_dir}")
        self.logger.debug(f"Initialized graphs directory at: {self.graphs}")

        self.app.add_route(f"{route_prefix}/verify_token", self.verify_botify_token, methods=['POST'])
        self.app.add_route(f"{route_prefix}/update_project", self.update_project, methods=['POST'])
        self.app.add_route(f"{route_prefix}/update_projects", self.update_projects, methods=['GET'])
        self.app.add_route(f"{route_prefix}/process_analysis", self.process_analysis, methods=['POST'])
        self.app.add_route(f"{route_prefix}/export_links", self.export_links, methods=['POST'])
        self.app.add_route(f"{route_prefix}/poll_export", self.poll_export, methods=['GET'])
        self.app.add_route(f"{route_prefix}/poll_meta_export", self.poll_meta_export, methods=['GET'])
        self.app.add_route(f"{route_prefix}/start_meta_export", self.start_meta_export, methods=['POST'])
        self.app.add_route(f"{route_prefix}/poll_links_export", self.poll_links_export, methods=['GET'])

        self.logger.debug("Initialized BotifyLinkGraph with prefix: {}", route_prefix)

    async def link_render(self):
        """Render the main link graph content."""
        global username, api_key

        try:
            if not api_key:
                return Card(
                    Header(
                        H3("API Key Required")
                    ),
                    P("Please go to 'Connect To Botify' in the menu to set up your API key first.")
                )

            org_projects = self.parse_candidates_file()
            self.logger.debug(f"Creating dropdown with {len(org_projects)} organizations")

            project_url_form = Form(
                Div(
                    Select(
                        Option("Select an organization...", value="", selected=True),
                        *[Option(org, value=org) for org in sorted(org_projects.keys())],
                        name="org_select",
                        id="org-select",
                        hx_get=f"{self.route_prefix}/update_projects",
                        hx_target="#project-select",
                        hx_trigger="change",
                        style="margin-bottom: 10px;"
                    ),
                    Select(
                        Option("Select a project...", value="", selected=True),
                        id="project-select",
                        name="project_select",
                        style="margin-bottom: 10px;"
                    ),
                    P("Or enter URL manually:", style="margin-bottom: 5px; color: #666;"),
                    Div(
                        Input(
                            type="text",
                            id="project-url-input",
                            name="project_url",
                            placeholder="Enter Botify project URL",
                            style="flex-grow: 1; margin-right: 10px; min-width: 300px;"
                        ),
                        Button(
                            "Get Analyses",
                            type="submit",
                            style="width: 200px;",
                        ),
                        Div(
                            hx_trigger="submit from:form",
                            hx_post="/chat",
                            hx_vals='{\"message\": \"Tell the user to select an Analysis Date then click Get Fields to pick their Link-Graph color-coded attributes...\"}',
                            hx_swap="none",
                            style="display: none;"
                        ),
                        style="display: flex; align-items: center; margin-bottom: 10px;"
                    ),
                    style="display: flex; flex-direction: column;"
                ),
                hx_post=f"{self.route_prefix}/update_project",
                hx_target="#update-status",
                hx_swap="outerHTML",
            )

            return Div(
                project_url_form,
                Div(id="update-status"),
                id=f"content-container{self.id_suffix}",
                style="display: flex; flex-direction: column;"
            )
        except ValueError:
            return P("Please go to 'Connect To Botify' in the menu to set up your API key first.")

    def check_for_file(self, org: str, project: str, analysis: str, type: str = "links") -> bool:
        """
        Checks if a file exists at the path:
        downloads/link-graph/{org}/{project}/{project}_{analysis}_{type}.csv

        Args:
            org (str): Organization name
            project (str): Project name 
            analysis (str): Analysis slug
            type (str): File type - either "links" or "meta"

        Returns:
            bool: True if file exists, False otherwise
        """
        if type not in ["links", "meta"]:
            raise ValueError("Type must be either 'links' or 'meta'")

        file_path = Path(f"downloads/link-graph/{org}/{project}/{project}_{analysis}_{type}.csv")
        return file_path.exists()

    def get_token(self) -> str:
        """Get Botify API token from file."""
        try:
            return self.token_file.read_text().strip()
        except Exception as e:
            self.logger.error(f"Error reading token file: {str(e)}")
            raise ValueError("Could not read Botify token. Please ensure botify_token.txt exists in the repository root.")

    async def get_user_profile(self, token: str) -> Optional[dict]:
        """Get user profile from Botify API."""
        headers = {"Authorization": f"Token {token}"}
        async with aiohttp.ClientSession() as session:
            try:
                async with session.get(self.auth_url, headers=headers) as response:
                    if response.status == 200:
                        return await response.json()
                    self.logger.error(f"Failed to get user profile. Status: {response.status}")
                    return None
            except Exception as e:
                self.logger.error(f"Error getting user profile: {str(e)}")
                return None

    async def find_optimal_depth(self, org, project, analysis, max_edges=1000000):
        """
        Determine the highest depth for which the number of edges does not exceed max_edges.
        """
        url = f"https://api.botify.com/v1/projects/{org}/{project}/query"
        previous_edges = 0
        headers = {
            "Authorization": f"Token {self.get_token()}",
            "Content-Type": "application/json"
        }

        await chat.broadcast("\nFinding optimal depth...\n")
        async with aiohttp.ClientSession() as session:
            for depth in range(1, 10):
                data_payload = {
                    "collections": [f"crawl.{analysis}"],
                    "query": {
                        "dimensions": [],
                        "metrics": [{"function": "sum", "args": [f"crawl.{analysis}.outlinks_internal.nb.total"]}],
                        "filters": {"field": f"crawl.{analysis}.depth", "predicate": "lte", "value": depth},
                    },
                }

                async with session.post(url, headers=headers, json=data_payload) as response:
                    data = await response.json()
                    edges = data["results"][0]["metrics"][0]

                    self.logger.debug(f"Depth {depth}: {edges:,} edges")
                    await chat.broadcast(f"Depth {depth}: {edges:,} edges\n")

                    if edges > max_edges or edges == previous_edges:
                        return (depth - 1 if depth > 1 else depth), previous_edges

                    previous_edges = edges

        return depth, previous_edges

    async def update_project(self, request):
        """Handle project URL submission and show analysis selection."""
        try:
            form = await request.form()
            project_url = form.get('project_url')
            self.current_project_url = project_url  # Store URL for later use

            # Extract org and project from URL
            parts = project_url.split('/')
            org = parts[-3]
            project = parts[-2]

            # Set up project directories
            project_dir = self.downloads_dir / org / project
            project_dir.mkdir(parents=True, exist_ok=True)
            self.graphs = project_dir

            self.logger.debug(f"Fetching analyses for {org}/{project}")

            # Fetch analyses using light endpoint
            url = f"{self.base_url}/analyses/{org}/{project}/light"
            headers = {
                "Authorization": f"Token {self.get_token()}",
                "Content-Type": "application/json"
            }

            async with aiohttp.ClientSession() as session:
                analyses = []
                async with session.get(url, headers=headers) as response:
                    if response.status != 200:
                        raise ValueError(f"Failed to fetch analyses: {response.status}")

                    data = await response.json()
                    analyses.extend(data.get('results', []))

                    # Handle pagination
                    while next_url := data.get('next'):
                        async with session.get(next_url, headers=headers) as next_response:
                            data = await next_response.json()
                            analyses.extend(data.get('results', []))

            if not analyses:
                return Div(
                    P("No analyses found for this project", style="color: red;"),
                    id="update-status"
                )

            # Sort analyses by slug (which contains the date)
            analyses.sort(key=lambda x: x.get('slug', ''), reverse=True)

            return Div(
                # CRITICAL: This card displays existing graph files and must maintain exact pattern:
                # - Glob pattern '*_links.csv' must match file generation pattern
                # - Download href must use exact '/download/{org}/{project}/{path.name}' format
                # - Target must be '_blank' to open in new tab
                # - List comprehension must preserve order of files
                Card(
                    H4("Existing Link Graphs"),
                    P(f"Found {len(list(self.graphs.glob('*links.csv')))} existing graph files"),
                    Ul(
                        *[Li(
                            A(
                                path.name,
                                href=f"/download/{org}/{project}/{path.name}",
                                target="_blank"
                            ),
                            " ",
                            A(
                                "(Link Graph)",
                                href=(
                                    f"https://cosmograph.app/run/?"
                                    f"data=http://localhost:5001/download/{org}/{project}/{path.name}"
                                    f"&meta=http://localhost:5001/download/{org}/{project}/{path.name.replace('_links.csv', '_meta.csv')}"
                                    f"&nodeColor=pagetype"
                                    if path.with_name(path.name.replace('_links.csv', '_meta.csv')).exists()
                                    else f"https://cosmograph.app/run/?data=http://localhost:5001/download/{org}/{project}/{path.name}"
                                ) + "&link-spring=.1",
                                target="_blank"
                            )
                        ) for path in self.graphs.glob('*_links.csv')]
                    ),
                    style="margin-bottom: 20px;"
                ),
                H4("Select Analysis"),
                Form(
                    Select(
                        *[Option(
                            # Determine display text based on which files exist
                            (f"{a.get('slug', 'Unknown')} (Links & Meta)"
                             if self.check_for_file(org, project, a.get('slug', 'Unknown'), "meta")
                             else f"{a.get('slug', 'Unknown')} (Links Only)"
                             if self.check_for_file(org, project, a.get('slug', 'Unknown'), "links")
                             else a.get('slug', 'Unknown')),
                            value=a.get('slug', ''),
                            disabled=self.check_for_file(org, project, a.get('slug', 'Unknown'), "links")
                        ) for a in analyses],
                        id="analysis-select",
                        name="analysis_select",
                        style="flex: 1; margin-right: 10px;"
                    ),
                    Button("Get Fields", type="submit", style="width: 180px;"),
                    hx_post=f"{self.route_prefix}/process_analysis",
                    hx_target="#analysis-results",
                    hx_indicator="#loading-indicator",
                    style="display: flex; align-items: center;"
                ),
                Div(
                    Img(src="/static/loading.gif", style="width: 50px;"),
                    "Processing...",
                    id="loading-indicator",
                    style="display: none;"
                ),
                Div(id="analysis-results"),
                id="update-status"
            )

        except Exception as e:
            self.logger.error(f"Error in update_project: {str(e)}")
            return Div(
                P(f"Error: {str(e)}", style="color: red;"),
                id="update-status"
            )

    async def process_analysis(self, request):
        """Process selected analysis and show available fields."""
        try:
            form = await request.form()
            analysis = form.get('analysis_select')

            if not analysis or not self.current_project_url:
                return P("Missing analysis or project URL", style="color: red;")

            # Extract org and project from stored URL
            parts = self.current_project_url.split('/')
            org = parts[-3]
            project = parts[-2]

            # Store current analysis info
            self.current_analysis = {'org': org, 'project': project, 'analysis': analysis}

            # Find optimal depth
            optimal_depth, edge_count = await self.find_optimal_depth(org, project, analysis)
            self.current_analysis['depth'] = optimal_depth

            # Define all available fields
            field_groups = {
                'Search Performance': {
                    'impressions': 'search_console.period_0.count_impressions',
                    'clicks': 'search_console.period_0.count_clicks'
                },
                'Page Type': {
                    'pagetype': f'crawl.{analysis}.segments.pagetype.value'
                },
                'Technical': {
                    'compliant': f'crawl.{analysis}.compliant.is_compliant',
                    'canonical': f'crawl.{analysis}.canonical.to.equal',
                    'sitemap': f'crawl.{analysis}.sitemaps.present'
                },
                # 'JavaScript': {
                #     'js_execution': f'crawl.{analysis}.js.rendering.exec',
                #     'js_success': f'crawl.{analysis}.js.rendering.ok'
                # }
            }

            return Card(
                H4("Select Color Maps"),
                P(f"Optimal depth: {optimal_depth} (approximately {edge_count:,} edges)"),
                Form(
                    # Field selection section
                    Fieldset(
                        Legend("Select Fields to Include"),
                        *[Div(
                            H5(group_name),
                            *[Div(
                                Input(
                                    type="checkbox",
                                    id=field_id,
                                    name=field_id,
                                    value=field_path,
                                    checked=True
                                ),
                                Label(
                                    field_id.replace('_', ' ').title(),
                                    _for=field_id,
                                    style="margin-left: 0.5rem;"
                                ),
                                style="margin: 0.5rem 0;"
                            ) for field_id, field_path in fields.items()],
                            style="margin: 1rem 0;"
                        ) for group_name, fields in field_groups.items()],
                        style="border: 1px solid #ddd; padding: 1rem; border-radius: 4px;"
                    ),
                    # Export button
                    Button(
                        "Start Export",
                        type="submit",
                        style="margin-top: 1rem;"
                    ),
                    hx_post=f"{self.route_prefix}/export_links",
                    hx_target="#export-status"
                ),
                Div(id="export-status"),
                style="margin-top: 1rem;"
            )

        except Exception as e:
            self.logger.error(f"Error processing analysis: {str(e)}")
            return P(f"Error: {str(e)}", style="color: red;")

    async def export_links(self, request):
        """Export the graph data to CSV."""
        try:
            form = await request.form()
            self.logger.debug(f"Export request form data: {form}")

            if not self.current_analysis:
                self.logger.error("No analysis selected for export")
                return P("No analysis selected", style="color: red;")

            # Store parameters needed for both exports
            self.export_params = {
                'org': self.current_analysis['org'],
                'project': self.current_analysis['project'],
                'analysis': self.current_analysis['analysis'],
                'depth': self.current_analysis['depth']
            }
            self.logger.info(f"Export parameters set: {self.export_params}")

            # Setup paths for both exports
            download_dir = self.downloads_dir / self.export_params['org'] / self.export_params['project']
            download_dir.mkdir(parents=True, exist_ok=True)

            self.links_download_path = download_dir / f"{self.export_params['project']}_{self.export_params['analysis']}_links.csv"
            self.meta_download_path = download_dir / f"{self.export_params['project']}_{self.export_params['analysis']}_meta.csv"
            self.logger.info(f"Download paths configured - Links: {self.links_download_path}, Meta: {self.meta_download_path}")

            # Store meta fields for later
            meta_fields = []
            # for field in ['impressions', 'clicks', 'pagetype', 'compliant', 'canonical', 'sitemap', 'js_execution', 'js_success']:
            for field in ['impressions', 'clicks', 'pagetype', 'compliant', 'canonical', 'sitemap']:
                if form.get(field):
                    meta_fields.append(form[field])
            self.meta_fields = meta_fields
            self.logger.info(f"Selected meta fields: {self.meta_fields}")

            # Start both exports in sequence
            links_response = await self.start_links_export(request)
            meta_response = await self.start_meta_export(request)

            return Div(
                links_response,
                meta_response,
                id="export-status"
            )

        except Exception as e:
            self.logger.error(f"Error in export process: {str(e)}", exc_info=True)
            return P(f"❌ Error: {str(e)}", style="color: red;")
    # End of Selection

    async def start_links_export(self, request):
        """Start the links export process."""
        try:
            # Construct links query
            query = {
                "dimensions": [
                    "url",
                    f"crawl.{self.export_params['analysis']}.outlinks_internal.graph.url"
                ],
                "metrics": [],
                "filters": {
                    "field": f"crawl.{self.export_params['analysis']}.depth",
                    "predicate": "lte",
                    "value": self.export_params['depth']
                }
            }
            self.logger.info(f"Constructed links query: {query}")

            data_payload = {
                "job_type": "export",
                "payload": {
                    "username": self.export_params['org'],
                    "project": self.export_params['project'],
                    "connector": "direct_download",
                    "formatter": "csv",
                    "export_size": 1000000,
                    "query": {
                        "collections": [f"crawl.{self.export_params['analysis']}"],
                        "query": query
                    }
                }
            }
            self.logger.info(f"Links export payload constructed: {data_payload}")

            headers = {"Authorization": f"Token {self.get_token()}", "Content-Type": "application/json"}

            async with aiohttp.ClientSession() as session:
                self.logger.info("Initiating links export API request")
                async with session.post(
                    "https://api.botify.com/v1/jobs",
                    json=data_payload,
                    headers=headers
                ) as response:
                    if response.status not in (200, 201):
                        error_text = await response.text()
                        self.logger.error(f"Links export API request failed with status {response.status}")
                        self.logger.error(f"Response body: {error_text}")
                        return P("❌ Links export failed to start", style="color: red;")

                    job_data = await response.json()
                    self.links_job_url = f"https://api.botify.com{job_data.get('job_url')}"
                    self.logger.info(f"Links export job created successfully with URL: {self.links_job_url}")

                    return Div(
                        P("✓ Links export started", style="color: green;"),
                        Div(id="meta-export-status"),
                        Div(
                            id="links-export-status",
                            hx_get=f"{self.route_prefix}/poll_links_export",
                            hx_trigger="load delay:3s"
                        ),
                        id="export-status"
                    )

        except Exception as e:
            self.logger.error(f"Error starting links export: {str(e)}", exc_info=True)
            return P(f"❌ Error: {str(e)}", style="color: red;")

    async def poll_links_export(self, request):
        """Poll the links export status."""
        try:
            if not hasattr(self, 'links_job_url'):
                self.logger.error("No links_job_url found - export may not have been initiated")
                return P("No links export job in progress", style="color: red;")

            headers = {"Authorization": f"Token {self.get_token()}", "Content-Type": "application/json"}

            async with aiohttp.ClientSession() as session:
                self.logger.info(f"Polling links job status at: {self.links_job_url}")
                async with session.get(self.links_job_url, headers=headers) as response:
                    if response.status != 200:
                        error_text = await response.text()
                        self.logger.error(f"Failed to check links job status. Status: {response.status}")
                        self.logger.error(f"Response: {error_text}")
                        return P("❌ Failed to check links job status", style="color: red;")

                    job_status = await response.json()
                    self.logger.info(f"Links job status response: {job_status}")

                    if job_status["job_status"] == "DONE":
                        download_url = job_status["results"]["download_url"]
                        self.logger.info(f"Links export completed. Downloading from: {download_url}")
                        relative_path = self.download_file(download_url, self.links_download_path)
                        self.logger.info(f"Links file downloaded successfully to: {relative_path}")

                        return Div(
                            P("✓ Links export completed", style="color: green;"),
                            P(f"Links data saved to: {relative_path}"),
                            id="links-export-status"
                        )

                    elif job_status["job_status"] == "FAILED":
                        self.logger.error(f"Links export job failed. Full status: {job_status}")
                        return Div(
                            P("❌ Links export failed", style="color: red;"),
                            id="links-export-status"
                        )

                    else:
                        self.logger.info(f"Links export in progress. Current status: {job_status['job_status']}")
                        return Div(
                            P(f"• Processing links export... Status: {job_status['job_status']}"),
                            id="links-export-status",
                            hx_get=f"{self.route_prefix}/poll_links_export",
                            hx_trigger="load delay:3s"
                        )

        except Exception as e:
            self.logger.error(f"Error polling links export: {str(e)}", exc_info=True)
            return P(f"❌ Error: {str(e)}", style="color: red;")

    async def start_meta_export(self, request):
        """Start the meta data export process."""
        try:
            form_data = await request.form()
            selected_fields = [v for v in form_data.values() if v != 'meta']

            # Separate dimensions and metrics
            dimensions = [f"crawl.{self.export_params['analysis']}.url"]
            metrics = []

            for field in selected_fields:
                if field.startswith('search_console'):
                    # Search console fields should be metrics, not dimensions
                    metrics.append(field)
                else:
                    dimensions.append(field)

            query = {
                'dimensions': dimensions,
                'metrics': metrics
            }

            payload = {
                'job_type': 'export',
                'payload': {
                    'username': self.export_params['org'],
                    'project': self.export_params['project'],
                    'connector': 'direct_download',
                    'formatter': 'csv',
                    'export_size': 1000000,
                    'query': {
                        'collections': [f"crawl.{self.export_params['analysis']}"]
                    }
                }
            }

            # Add search console collection if needed
            if metrics:
                payload['payload']['query']['collections'].append('search_console')
                payload['payload']['query']['periods'] = [[
                    self.export_params['analysis'][:4] + '-' +
                    self.export_params['analysis'][4:6] + '-' +
                    self.export_params['analysis'][6:],
                    self.export_params['analysis'][:4] + '-' +
                    self.export_params['analysis'][4:6] + '-' +
                    self.export_params['analysis'][6:]
                ]]

            # Add query after collections are set
            payload['payload']['query']['query'] = query

            self.logger.info(f"Final meta export payload: {payload}")

            headers = {"Authorization": f"Token {self.get_token()}", "Content-Type": "application/json"}

            async with aiohttp.ClientSession() as session:
                self.logger.info("Initiating meta export API request")
                async with session.post(
                    "https://api.botify.com/v1/jobs",
                    json=payload,
                    headers=headers
                ) as response:
                    if response.status not in (200, 201):
                        error_text = await response.text()
                        self.logger.error(f"Meta export failed to start with status {response.status}")
                        self.logger.error(f"Response headers: {response.headers}")
                        self.logger.error(f"Response body: {error_text}")
                        return P("❌ Meta export failed to start", style="color: red;")

                    job_data = await response.json()
                    self.meta_job_url = f"https://api.botify.com{job_data.get('job_url')}"
                    self.logger.info(f"Meta export job created successfully with URL: {self.meta_job_url}")

                    return Div(
                        P("✓ Meta export started", style="color: green;"),
                        Div(
                            id="meta-export-status",
                            hx_get=f"{self.route_prefix}/poll_meta_export",
                            hx_trigger="load delay:3s"
                        ),
                        id="export-status"
                    )

        except Exception as e:
            self.logger.error(f"Error starting meta export: {str(e)}", exc_info=True)
            return P(f"❌ Error: {str(e)}", style="color: red;")

    async def poll_export(self, request):
        """
        Poll the export status and handle different stages.

        This method follows a strict sequential process:
        1. Check if export job exists and get its status
        2. If DONE, download and decompress the _links file
        3. Only after successful _links download, trigger _meta export
        4. Poll _meta export status 
        5. Download _meta file when ready

        No concurrent operations are allowed - each step must complete
        before the next can begin.

        IMPORTANT SEQUENCING NOTE:
        The _meta export/download sequence that follows the _links download
        must maintain this strict ordering to prevent race conditions or
        incomplete data. Any implementation attempting concurrent operations
        or reordering these steps will be rejected as it violates core
        reliability requirements.
        """
        self.logger.debug("Poll export state:")
        self.logger.debug(f"Job URL: {self.job_url}")

        # Add safety check for export_path
        if not hasattr(self, 'export_path'):
            self.logger.warning("Export path not set - export may not have been initiated properly")
            return P("Export not properly initialized", style="color: red;")

        self.logger.debug(f"Export path: {self.export_path}")

        try:
            if not hasattr(self, 'job_url'):
                self.logger.warning("No job_url attribute found")
                return P("No export job in progress", style="color: red;")

            if not hasattr(self, 'export_params'):
                self.logger.warning("No export_params attribute found")
                return P("Export parameters not found", style="color: red;")

            headers = {"Authorization": f"Token {self.get_token()}", "Content-Type": "application/json"}

            async with aiohttp.ClientSession() as session:
                # Step 1: Check current job status
                self.logger.debug(f"Making GET request to job URL: {self.job_url}")
                async with session.get(self.job_url, headers=headers) as response:
                    self.logger.debug(f"Job status response code: {response.status}")

                    if response.status != 200:
                        response_text = await response.text()
                        self.logger.error(f"Failed to check job status. Status: {response.status}, Response: {response_text}")
                        return P("❌ Failed to check job status", style="color: red;")

                    job_status = await response.json()
                    self.logger.debug(f"Full job status response: {job_status}")
                    self.logger.debug(f"Job status state: {job_status['job_status']}")

                    if job_status["job_status"] == "DONE":
                        self.logger.info("Job completed successfully")
                        # Step 2: Download and process _links file
                        download_url = job_status["results"]["download_url"]
                        self.logger.debug(f"Download URL from job: {download_url}")

                        # Use the established download path
                        self.logger.debug(f"Attempting download to: {self.current_download_path}")

                        # Download file using aiohttp
                        async with session.get(download_url, headers=headers) as download_response:
                            if download_response.status != 200:
                                self.logger.error(f"Failed to download file. Status: {download_response.status}")
                                return P("❌ Failed to download file", style="color: red;")

                            # Ensure parent directories exist
                            self.current_download_path.parent.mkdir(parents=True, exist_ok=True)

                            # Write file asynchronously
                            with open(self.current_download_path, 'wb') as f:
                                while True:
                                    chunk = await download_response.content.read(8192)
                                    if not chunk:
                                        break
                                    write(chunk)

                        self.logger.info(f"File downloaded successfully to: {self.current_download_path}")
                        relative_path = self.current_download_path.relative_to(self.downloads_dir.parent)

                        # Step 3: Validate meta data availability
                        url = f"https://api.botify.com/v1/projects/{self.export_params['org']}/{self.export_params['project']}/query"
                        self.logger.debug(f"Validating meta data availability at: {url}")

                        self.logger.debug(f"Attempting meta data validation on file: {self.export_path}")
                        self.logger.debug(f"File exists: {os.path.exists(self.export_path)}")
                        self.logger.debug(f"File size: {os.path.getsize(self.export_path) if os.path.exists(self.export_path) else 'N/A'}")

                        # Build validation query directly
                        validation_query = {
                            "fields": self.selected_fields,
                            "filters": {
                                "and": [
                                    {"field": "url", "predicate": "in", "value": "_links"}
                                ]
                            }
                        }

                        # Add search console specific criteria if needed
                        if any('search_console' in field for field in self.selected_fields):
                            # Get date range for search console data
                            validation_query.update({
                                "collections": [
                                    f"crawl.{self.export_params['analysis']}",
                                    "search_console"
                                ],
                                "periods": [
                                    [
                                        # Use a 30 day window ending at analysis date
                                        f"{self.export_params['analysis'][:4]}-{self.export_params['analysis'][4:6]}-{self.export_params['analysis'][6:8]}",
                                        f"{self.export_params['analysis'][:4]}-{self.export_params['analysis'][4:6]}-{self.export_params['analysis'][6:8]}"
                                    ]
                                ]
                            })
                        self.logger.debug(f"Validation query before builder: {validation_query}")
                        validation_query = query_builder.build_validation_query()
                        self.logger.debug(f"Meta validation query after builder: {validation_query}")

                        async with session.post(url, headers=headers, json=validation_query) as validation_response:
                            validation_status = validation_response.status
                            validation_text = await validation_response.text()

                            if validation_status == 200:
                                data = await validation_response.json()
                                row_count = len(data.get('results', []))
                                self.logger.info(f"Meta export would return {row_count} rows")

                                return Div(
                                    P("✓ Links export completed", style="color: green;"),
                                    P("✓ Links download completed", style="color: green;"),
                                    P(f"Links data saved to: {relative_path}"),
                                    P(f"✓ Meta data available: {row_count} rows", style="color: green;"),
                                    Button("Start Meta Export",
                                           hx_post=f"{self.route_prefix}/start_meta_export",
                                           style="background-color: #4CAF50; color: white;"),
                                    id="meta-export-status"
                                )
                            else:
                                self.logger.warning(f"Meta data validation failed with status {validation_status}: {validation_text}")
                                return Div(
                                    P("✓ Links export completed", style="color: green;"),
                                    P("✓ Links download completed", style="color: green;"),
                                    P(f"Links data saved to: {relative_path}"),
                                    P("❌ Meta data not available", style="color: red;"),
                                    id="export-status"
                                )

                    elif job_status["job_status"] == "FAILED":
                        self.logger.error(f"Job failed with status: {job_status}")
                        if 'error' in job_status:
                            self.logger.error(f"Error details: {job_status['error']}")
                        return Div(
                            P("❌ Export failed", style="color: red;"),
                            id="export-status"
                        )

                    else:
                        self.logger.debug(f"Job still in progress, status: {job_status['job_status']}")
                        return Div(
                            P(f"• Processing links export... Status: {job_status['job_status']}"),
                            id="export-status",
                            hx_get=f"{self.route_prefix}/poll_export",
                            hx_trigger="load delay:3s"
                        )

        except Exception as e:
            self.logger.error(f"Error in export process: {str(e)}", exc_info=True)
            return P(f"❌ Error: {str(e)}", style="color: red;")

    async def poll_meta_export(self, request):
        """Poll meta export status and handle completion."""
        try:
            if not hasattr(self, 'meta_job_url'):
                self.logger.warning("No meta export job URL found")
                return P("No meta export job in progress", style="color: red;")

            headers = {"Authorization": f"Token {self.get_token()}", "Content-Type": "application/json"}

            async with aiohttp.ClientSession() as session:
                # Check current job status
                self.logger.debug(f"Making GET request to meta job URL: {self.meta_job_url}")
                async with session.get(self.meta_job_url, headers=headers) as response:
                    if response.status != 200:
                        response_text = await response.text()
                        self.logger.error(f"Failed to check meta job status. Status: {response.status}, Response: {response_text}")
                        return P("❌ Failed to check meta job status", style="color: red;")

                    job_status = await response.json()
                    self.logger.debug(f"Meta job status: {job_status}")

                    if job_status["job_status"] == "DONE":
                        self.logger.info("Meta export completed successfully")
                        download_url = job_status["results"]["download_url"]

                        # Download to temporary gzip file first
                        temp_gz = self.meta_download_path.with_suffix('.gz')

                        async with session.get(download_url, headers=headers) as download_response:
                            if download_response.status != 200:
                                self.logger.error(f"Failed to download meta file. Status: {download_response.status}")
                                return P("❌ Failed to download meta file", style="color: red;")

                            # Ensure parent directories exist
                            self.meta_download_path.parent.mkdir(parents=True, exist_ok=True)

                            # Write compressed file
                            with open(temp_gz, 'wb') as gz_file:
                                while True:
                                    chunk = await download_response.content.read(8192)
                                    if not chunk:
                                        break
                                    gz_file.write(chunk)

                            # Decompress
                            with gzip.open(temp_gz, 'rb') as f_in:
                                with open(self.meta_download_path, 'wb') as f_out:
                                    shutil.copyfileobj(f_in, f_out)

                            # Cleanup
                            temp_gz.unlink()

                        self.logger.info(f"Meta file downloaded successfully to: {self.meta_download_path}")

                        try:
                            # Read the downloaded CSV
                            df = pd.read_csv(self.meta_download_path)

                            # Set the column names in the exact order needed
                            df.columns = ['id', 'pagetype', 'is_compliant', 'canonical_equal', 'sitemaps_present', 'impressions', 'clicks']

                            # Save the processed DataFrame back to CSV
                            df.to_csv(self.meta_download_path, index=False)

                            self.logger.info(f"Meta file processed and saved successfully to: {self.meta_download_path}")
                            relative_path = self.meta_download_path.relative_to(self.downloads_dir.parent)

                            return Div(
                                P("✓ Meta export completed", style="color: green;"),
                                P("✓ Meta download completed", style="color: green;"),
                                P(f"Meta data saved to: {relative_path}"),
                                id="meta-export-status"
                            )

                        except pd.errors.EmptyDataError:
                            self.logger.error("Downloaded meta file is empty")
                            return P("❌ Meta file is empty", style="color: red;")
                        except Exception as e:
                            self.logger.error(f"Error processing meta file: {str(e)}", exc_info=True)
                            return P(f"❌ Error processing meta file: {str(e)}", style="color: red;")

                    elif job_status["job_status"] == "FAILED":
                        self.logger.error(f"Meta export failed with status: {job_status}")
                        if 'error' in job_status:
                            self.logger.error(f"Error details: {job_status['error']}")
                        return Div(
                            P("❌ Meta export failed", style="color: red;"),
                            id="meta-export-status"
                        )

                    else:
                        self.logger.debug(f"Meta export still in progress, status: {job_status['job_status']}")
                        return Div(
                            P(f"• Processing meta export... Status: {job_status['job_status']}"),
                            id="meta-export-status",
                            hx_get=f"{self.route_prefix}/poll_meta_export",
                            hx_trigger="load delay:3s"
                        )

        except Exception as e:
            self.logger.error(f"Error in meta export process: {str(e)}", exc_info=True)
            return P(f"❌ Error: {str(e)}", style="color: red;")

    def download_file(self, url: str, save_path: Path):
        """
        Download helper that saves files to the correct location.

        IMPORTANT PATH STRUCTURE:
        - Base directory is set in botifymograph.py as:
        self.downloads_dir = Path("downloads") / route_prefix.lstrip("/")

        - Files are then organized as:
        downloads/link-graph/{org}/{project}/{project}_{analysis}_links.csv

        This structure must be maintained even if looking at other implementations
        that may use different conventions.
        """
        self.logger.debug(f"Downloading from {url} to {save_path}")

        # Ensure parent directories exist
        save_path.parent.mkdir(parents=True, exist_ok=True)

        # Check if this is a links or meta file based on filename
        stem = save_path.stem  # Get filename without extension
        if stem.endswith('_links'):
            self.logger.debug("Detected links file")
            file_type = 'links'
        elif stem.endswith('_meta'):
            self.logger.debug("Detected meta file")
            file_type = 'meta'
        else:
            self.logger.warning(f"Unknown file type for {save_path}")
            file_type = 'unknown'

        # Download to temporary .gz file first
        temp_gz_path = save_path.with_suffix(save_path.suffix + '.gz')
        response = requests.get(url, stream=True)
        with open(temp_gz_path, "wb") as file:
            for chunk in response.iter_content(chunk_size=8192):
                file.write(chunk)

        # Decompress .gz file to final path and remove temp file
        with gzip.open(temp_gz_path, 'rb') as gz_file:
            with open(save_path, 'wb') as out_file:
                out_file.write(gz_file.read())

        temp_gz_path.unlink()  # Remove temporary .gz file

        self.logger.debug(f"File downloaded and decompressed successfully to {save_path}")

        # Add appropriate column names based on file type
        if file_type == 'links':
            df = pd.read_csv(save_path)
            df.columns = ['source', 'target']
            df.to_csv(save_path, index=False)
        elif file_type == 'meta':
            df = pd.read_csv(save_path)
            df.columns = ['id', 'pagetype', 'is_compliant', 'canonical_equal', 'sitemaps_present', 'impressions', 'clicks']
            df.to_csv(save_path, index=False)

        # Return relative path and file type
        return save_path.relative_to(self.downloads_dir.parent), file_type

    def parse_candidates_file(self):
        """Parse the candidates.txt file into a structured org/project dict."""
        candidates_path = Path('../botifybyob/candidates.txt')
        if not candidates_path.exists():
            self.logger.warning("Candidates file not found")
            return {}

        self.logger.debug(f"Reading candidates from {candidates_path}")

        org_projects = {}
        current_org = None

        for line in candidates_path.read_text().splitlines():
            if not line.strip():
                continue
            if not line.startswith(' '):
                current_org = line.strip()
                org_projects[current_org] = []
            else:
                if current_org:
                    project = line.strip()
                    # Skip projects containing "ad hoc" or "adhoc" (case insensitive)
                    if "ad hoc" not in project.lower() and "adhoc" not in project.lower():
                        org_projects[current_org].append(project)

        self.logger.debug(f"Found {len(org_projects)} organizations")
        return org_projects

    async def update_projects(self, request):
        """Handle updating the projects dropdown based on org selection."""
        org = request.query_params.get('org_select')
        if not org:
            return Select(
                Option("Select a project...", value="", selected=True),
                id="project-select",
                name="project_select"
            )

        org_projects = self.parse_candidates_file()
        projects = org_projects.get(org, [])

        return Select(
            Option("Select a project...", value="", selected=True),
            *[Option(proj, value=proj) for proj in sorted(projects)],
            id="project-select",
            name="project_select",
            onchange="""
                const org = document.getElementById('org-select').value;
                const proj = this.value;
                if (org && proj) {
                    const url = `https://app.botify.com/${org}/${proj}/`;
                    console.log('Setting URL to:', url);
                    document.getElementById('project-url-input').value = url;
                }
            """,
            style="margin-bottom: 10px;"
        )

    async def verify_botify_token(self, request):
        """Handle Botify token verification."""
        form = await request.form()
        token = form.get('botify_token')

        user_profile = await self.get_user_profile(token)

        if user_profile:
            username = user_profile.get("data", {}).get("username", "User")
            self.token_file.write_text(token)
            db["username"] = username
            return self.create_integration_card(username=username)
        else:
            return self.create_integration_card(error_message="Failed to verify the Botify token. Please try again.")

    def create_integration_card(self, username=None, error_message=None):
        """Create the Botify integration card."""
        if username:
            return Article(
                Header(
                    H3("Botify API Connected", style="margin: 0;"),
                ),
                P(f"Welcome, {username.title().replace('.', ' ')}! The Botify API is active."),
                P("You can now use the Link Graph features."),
                id=f"botify-integration-card{self.id_suffix}",
                style="border-radius: 5px; margin: 10px 0;"
            )
        else:
            return Card(
                H3("Botify Integration"),
                P("Botify integration is not set up. Please add your Botify token to use the Link Graph features."),
                P("Botify Employee? ", A("Find your token here", href="https://app.botify.com/account/", target="_blank")),
                P(error_message, style="color: red;") if error_message else None,
                Form(
                    Div(
                        Input(
                            type="text",
                            name="botify_token",
                            placeholder="Paste your Botify API token here",
                            required=True,
                            style="flex-grow: 1; margin-right: 10px;"
                        ),
                        Button("Verify Token", type="submit"),
                        style="display: flex; align-items: center; margin-bottom: 10px;"
                    ),
                    hx_post=f"{self.route_prefix}/verify_token",
                    hx_target=f"#botify-integration-card{self.id_suffix}",
                    hx_swap="outerHTML"
                ),
                id=f"botify-integration-card{self.id_suffix}",
                style="margin-top: 20px;"
            )

    async def check_download_status(self, job_id: str) -> dict:
        """Check status of a download job"""
        url = f"https://api.botify.com/v1/jobs/{job_id}"

        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(url) as response:
                    if response.status != 200:
                        return {"status": "error", "message": f"Status check failed: {response.status}"}

                    result = await response.json()
                    status = result.get("status", "unknown")

                    if status == "success":
                        return {
                            "status": "success",
                            "url": result.get("file_url"),
                            "expires": result.get("file_expires_at")
                        }
                    elif status == "error":
                        return {
                            "status": "error",
                            "message": result.get("error", "Unknown error")
                        }
                    else:
                        return {
                            "status": "pending",
                            "progress": result.get("progress", 0)
                        }

        except Exception as e:
            self.logger.error(f"Status check failed: {str(e)}")
            return {"status": "error", "message": str(e)}


link_graph = BotifyLinkGraph(app, route_prefix="/link-graph", id_suffix="")


@app.get("/download/{org}/{project}/{filename}")
async def host_file(request):
    """Serve CSV files from the downloads directory with CORS headers"""
    org = request.path_params['org']
    project = request.path_params['project']
    filename = request.path_params['filename']

    # Remove any query parameters from filename
    if '?' in filename:
        filename = filename.split('?')[0]

    # Try both direct path and link-graph subdirectory
    file_path = Path("downloads") / org / project / filename
    link_graph_path = Path("downloads") / "link-graph" / org / project / filename

    headers = {
        "Access-Control-Allow-Origin": "*",
        "Access-Control-Allow-Methods": "*",
        "Access-Control-Allow-Headers": "*"
    }

    if file_path.exists():
        return FileResponse(file_path, filename=filename, headers=headers)
    elif link_graph_path.exists():
        return FileResponse(link_graph_path, filename=filename, headers=headers)

    return PlainTextResponse(f"File not found at {file_path} or {link_graph_path}", status_code=404)