Orchestrating the Forever Machine: Automating the Knowledge Pipeline

AI_PHOOEY_CHOP = """\
# # PURPOSE OF CONTEXT: To show AI all the publishing steps for consolidating some into a master script
/home/mike/repos/MikeLev.in/_posts/2025-12-19-amnesiac-genie-storytelling-ai-future-proofing.md
scripts/articles/editing_prompt.txt
scripts/articles/contextualizer.py
scripts/articles/generate_semrush_candidates.py
scripts/articles/gsc_historical_fetch.py
scripts/articles/build_hierarchy.py
scripts/articles/build_navgraph.py
scripts/articles/generate_hubs.py
scripts/articles/show_graph.html
"""

# scripts/articles/update_graphs.py
import os
import sys
import subprocess
import time
from pathlib import Path

# --- CONFIGURATION ---
# We assume sibling scripts are in the same directory as this orchestrator
BASE_DIR = Path(__file__).parent.resolve()

## Final Verdict: The Orchestration of Reality

# The Sequence of Operations (Order Matters!)
PIPELINE = [
    {
        "name": "Contextualizer",
        "file": "contextualizer.py",
        "desc": "Extracting keywords and metadata from new articles..."
    },
    {
        "name": "SEMRush Candidates",
        "file": "generate_semrush_candidates.py",
        "desc": "Updating keyword candidate list for market analysis..."
    },
    {
        "name": "GSC Historical Fetch",
        "file": "gsc_historical_fetch.py",
        "desc": "Fetching latest performance velocity from Google Search Console..."
    },
    {
        "name": "Hierarchy Builder",
        "file": "build_hierarchy.py",
        "desc": "Clustering content and generating D3 Link Graph..."
    },
    {
        "name": "NavGraph Builder",
        "file": "build_navgraph.py",
        "desc": "Constructing the recursive navigation tree (NavGraph)..."
    },
    {
        "name": "Hub Generator",
        "file": "generate_hubs.py",
        "desc": "Generating static Jekyll Hub pages from NavGraph..."
    }
]

def run_step(step_info):
    """Runs a single python script as a subprocess."""
    script_path = BASE_DIR / step_info["file"]
    
    if not script_path.exists():
        print(f"❌ ERROR: Could not find {script_path}")
        return False

    print(f"\n--- 🚀 Step: {step_info['name']} ---")
    print(f"ℹ️  {step_info['desc']}")
    
    try:
        # We use sys.executable to ensure we use the same Python interpreter (and venv)
        # that is running this orchestrator.
        start_time = time.time()
        result = subprocess.run(
            [sys.executable, str(script_path)], 
            check=True,
            text=True
        )
        duration = time.time() - start_time
        print(f"✅ {step_info['name']} complete ({duration:.2f}s).")
        return True
        
    except subprocess.CalledProcessError as e:
        print(f"\n💥 ERROR in {step_info['name']}!")
        print(f"   Exit Code: {e.returncode}")
        return False
    except Exception as e:
        print(f"\n💥 UNEXPECTED ERROR: {e}")
        return False

def main():
    print(f"🤖 Initiating Pipulate Graph Update Sequence...")
    print(f"📂 Working Directory: {BASE_DIR}")
    
    total_start = time.time()
    success_count = 0
    
    for step in PIPELINE:
        if run_step(step):
            success_count += 1
        else:
            print("\n🛑 Pipeline halted due to error.")
            sys.exit(1)

    total_duration = time.time() - total_start
    print(f"\n✨ All {success_count} steps completed successfully in {total_duration:.2f}s.")
    print("👉 Your Link Graph and Hub Pages are now synchronized with Reality.")

if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
The Dragnet: Fetches historical GSC data to ground the Virtual Graph in Reality.
Aggregates 16 months of data to eliminate 'Gray Circles' (Unknown Status).

Outputs: gsc_velocity.json (Rate-limited to once per day)
"""

import os
import sys
import json
import time
import re
import argparse
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from google.oauth2 import service_account
from googleapiclient.discovery import build
import random
from pathlib import Path

# --- CONFIGURATION ---
SITE_URL = "sc-domain:mikelev.in" 
SCRIPT_DIR = Path(__file__).parent.resolve()
# Adjust path to match your actual key location provided in context
SERVICE_ACCOUNT_KEY_FILE = Path.home() / ".config/articleizer/service-account-key.json"
OUTPUT_FILE = SCRIPT_DIR / 'gsc_velocity.json'

# The date of the "Crash" to pivot analysis around
CRASH_DATE = datetime(2025, 4, 23).date()
HISTORY_MONTHS = 16
ROW_LIMIT = 25000  # Max allowed by API per request

SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly']

def authenticate_gsc():
    """Authenticates with the GSC API using Service Account."""
    if not SERVICE_ACCOUNT_KEY_FILE.exists():
        print(f"❌ Key file not found: {SERVICE_ACCOUNT_KEY_FILE}")
        sys.exit(1)
    creds = service_account.Credentials.from_service_account_file(
        SERVICE_ACCOUNT_KEY_FILE, scopes=SCOPES)
    return build('webmasters', 'v3', credentials=creds)

def extract_slug(url):
    """
    Normalizes a full URL into the specific slug format expected by build_hierarchy.py.
    """
    # Remove protocol and domain
    path = url.replace(SITE_URL.replace("sc-domain:", "https://"), "")
    path = path.replace("http://", "").replace("https://", "")
    
    # Remove domain if still present (for sc-domain properties)
    if "/" in path:
        path = path.split("/", 1)[1] 
    
    # Strip slashes
    clean_path = path.strip("/")
    
    # Get the last segment (the slug)
    if "/" in clean_path:
        slug = clean_path.split("/")[-1]
    else:
        slug = clean_path
        
    return slug

def fetch_month_data(service, start_date, end_date):
    """Fetches clicks per page for a specific date range with heavy pagination."""
    request = {
        'startDate': start_date.strftime('%Y-%m-%d'),
        'endDate': end_date.strftime('%Y-%m-%d'),
        'dimensions': ['page'],
        'rowLimit': ROW_LIMIT,
        'startRow': 0
    }
    
    all_rows = []
    
    while True:
        try:
            response = service.searchanalytics().query(siteUrl=SITE_URL, body=request).execute()
            rows = response.get('rows', [])
            all_rows.extend(rows)
            
            # Check if we hit the limit, if so, page next
            if len(rows) == ROW_LIMIT:
                print(".", end="", flush=True) # visual heartbeat
                request['startRow'] += ROW_LIMIT
                time.sleep(0.5) # Be nice to the API
            else:
                break
                
        except Exception as e:
            if "429" in str(e) or "Quota" in str(e):
                print(f"⏳", end="", flush=True)
                time.sleep(5) # Backoff
                continue
            print(f"\n  ⚠️ Error fetching {start_date}: {e}")
            break

    # Convert to dict: slug -> clicks (Aggregating if slugs duplicate due to protocol variations)
    mapped_data = {}
    for r in all_rows:
        url = r['keys'][0]
        clicks = r['clicks']
        slug = extract_slug(url)
        
        if slug:
            if slug in mapped_data:
                mapped_data[slug] += clicks
            else:
                mapped_data[slug] = clicks
            
    return mapped_data

def should_run(force=False):
    """Checks if the output file exists and was updated today."""
    if force:
        return True
        
    if not OUTPUT_FILE.exists():
        return True
        
    try:
        with open(OUTPUT_FILE, 'r', encoding='utf-8') as f:
            data = json.load(f)
            
        last_updated = data.get('_meta', {}).get('last_updated')
        today = datetime.now().strftime('%Y-%m-%d')
        
        if last_updated == today:
            print(f"✅ GSC Data is fresh for today ({today}). Skipping fetch.")
            return False
            
    except Exception:
        # If file is corrupt or format is old, run anyway
        return True
        
    return True

def main():
    parser = argparse.ArgumentParser(description="Fetch GSC History")
    parser.add_argument('--force', action='store_true', help="Ignore cache and force fetch")
    args = parser.parse_args()

    if not should_run(args.force):
        return

    print(f"🚀 Starting GSC Historical Dragnet for {SITE_URL}")
    print(f"📅 Pivot Date (Crash): {CRASH_DATE}")
    
    service = authenticate_gsc()
    
    # Generate date ranges (Monthly chunks going back 16 months)
    # We lag 3 days because GSC data is never real-time
    end_date = datetime.now().date() - timedelta(days=3) 
    current = end_date
    
    # Data Structure:
    # { 
    #   "_meta": { "last_updated": "YYYY-MM-DD" },
    #   "my-slug": { ... } 
    # }
    history_data = {} 
    
    print(f"⏳ Fetching last {HISTORY_MONTHS} months of data...")
    
    total_months_processed = 0
    
    for _ in range(HISTORY_MONTHS):
        # Calculate month window
        month_end = current
        month_start = (current - relativedelta(months=1)) + timedelta(days=1)
        
        month_key = month_start.strftime('%Y-%m')
        print(f"  [{month_key}] Fetching...", end="", flush=True)
        
        data = fetch_month_data(service, month_start, month_end)
        
        page_count = len(data)
        click_count = sum(data.values())
        print(f" ✓ {page_count} pages / {click_count:.0f} clicks")
        
        # Merge into main history
        for slug, clicks in data.items():
            if slug not in history_data:
                history_data[slug] = {'timeline': {}}
            
            # Add to timeline
            history_data[slug]['timeline'][month_key] = clicks
            
        current = month_start - timedelta(days=1)
        total_months_processed += 1
        time.sleep(random.uniform(0.5, 1.5)) # Human jitter

    print(f"\n🧮 Calculating Velocity and Health Scores for {len(history_data)} unique slugs...")
    
    final_output = {
        "_meta": {
            "last_updated": datetime.now().strftime('%Y-%m-%d')
        }
    }
    
    for slug, data in history_data.items():
        timeline = data['timeline']
        sorted_months = sorted(timeline.keys())
        
        # Calculate Pre/Post Crash Averages
        pre_crash_clicks = []
        post_crash_clicks = []
        recent_clicks = [] # Last 3 months for velocity
        
        # Calculate recent threshold date
        recent_threshold = (end_date - relativedelta(months=3))
        
        for month_str in sorted_months:
            m_date = datetime.strptime(month_str, '%Y-%m').date()
            clicks = timeline[month_str]
            
            if m_date < CRASH_DATE.replace(day=1):
                pre_crash_clicks.append(clicks)
            else:
                post_crash_clicks.append(clicks)
                
            if m_date >= recent_threshold.replace(day=1):
                recent_clicks.append(clicks)
        
        avg_pre = sum(pre_crash_clicks) / len(pre_crash_clicks) if pre_crash_clicks else 0
        avg_post = sum(post_crash_clicks) / len(post_crash_clicks) if post_crash_clicks else 0
        
        # Velocity: Slope of last 3 months
        velocity = 0
        if len(recent_clicks) >= 2:
            # Simple diff: Latest month - 3 months ago
            velocity = recent_clicks[-1] - recent_clicks[0]
            
        # Health Status Determination
        status = "stable"
        
        if avg_pre > 0:
            recovery_ratio = avg_post / avg_pre
            if recovery_ratio < 0.5:
                status = "critical"   # Lost >50% traffic
            elif recovery_ratio < 0.8:
                status = "ailing"     # Lost >20% traffic
            elif recovery_ratio > 1.2:
                status = "thriving"   # Grew >20%
            elif recovery_ratio > 0.8:
                status = "recovering" # Holding steady-ish
        elif avg_post > 5: # Low threshold for "Newborn"
            status = "newborn" # No pre-crash data, but has traffic now
        elif avg_post == 0 and avg_pre == 0:
            status = "unknown"
        elif avg_post == 0:
            status = "dormant"

        final_output[slug] = {
            "total_clicks": sum(timeline.values()),
            "pre_crash_avg": round(avg_pre, 1),
            "post_crash_avg": round(avg_post, 1),
            "velocity": velocity,
            "status": status,
            "timeline": timeline
        }

    # Save to file
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(final_output, f, indent=2)
        
    print(f"💾 Saved velocity data to {OUTPUT_FILE}")
    print(f"💎 Total Unique Content Nodes Grounded: {len(history_data)}")

if __name__ == "__main__":
    main()

import json
import glob
from pathlib import Path
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import MiniBatchKMeans
from sklearn.decomposition import TruncatedSVD
from collections import Counter
import re
import warnings

# Silence the specific warning if copy usage is correct logic-wise
warnings.filterwarnings('ignore', category=pd.errors.SettingWithCopyWarning)

# --- CONFIGURATION ---
# Adjust path to your context folder relative to script execution location
CONTEXT_DIR = Path("../../_posts/_context") 
OUTPUT_FILE = "graph.json"
TARGET_BRANCHING_FACTOR = 7  # The "Rule of 7" (Clusters)
GOLD_PAN_SIZE = 5            # Number of "Top Articles" to keep at the Hub level
MIN_CLUSTER_SIZE = 5         # Don't split if smaller than this (Restored!)

def load_shards(directory):
    """Ingests the Holographic Shards (JSON context files)."""
    shards = []
    # Handle relative path resolution if run from different dir
    if not directory.exists():
         # Fallback try relative to this file
         directory = Path(__file__).parent / directory
    
    files = list(directory.glob("*.json"))
    print(f"💎 Found {len(files)} shards in {directory}...")
    
    for f in files:
        try:
            with open(f, 'r', encoding='utf-8') as file:
                data = json.load(file)
                # Create a rich semantic soup for vectorization
                # Weighting: Title (3x), Keywords (2x), Subtopics (1x)
                soup = (
                    (data.get('t', '') + " ") * 3 + 
                    (" ".join(data.get('kw', [])) + " ") * 2 + 
                    " ".join(data.get('sub', []))
                )
                
                shards.append({
                    "id": data.get('id', f.stem),
                    "label": data.get('t', 'Untitled'),
                    "soup": soup,
                    "keywords": data.get('kw', []) + data.get('sub', []), # For labeling
                    "type": "article"
                })
        except Exception as e:
            print(f"⚠️ Error loading {f.name}: {e}")
            
    return pd.DataFrame(shards)

def load_market_data(directory=Path(".")):
    """Loads SEMRush/GSC CSV data for gravity weighting."""
    if not directory.exists():
        directory = Path(__file__).parent

    files = list(directory.glob("*bulk_us*.csv"))
    if not files:
        print("ℹ️ No market data (CSV) found. Graph will be unweighted.")
        return {}
    
    latest_file = max(files, key=lambda f: f.stat().st_mtime)
    print(f"💰 Loading market gravity from: {latest_file.name}")
    
    try:
        df = pd.read_csv(latest_file)
        market_map = {}
        for _, row in df.iterrows():
            kw = str(row['Keyword']).lower().strip()
            try:
                vol = int(row['Volume'])
            except:
                vol = 0
            market_map[kw] = vol
        return market_map
    except Exception as e:
        print(f"⚠️ Error loading market data: {e}")
        return {}

def load_velocity_data(directory=Path(".")):
    """Loads GSC velocity/health data."""
    if not directory.exists():
        directory = Path(__file__).parent
        
    velocity_file = directory / "gsc_velocity.json"
    if not velocity_file.exists():
        print("ℹ️ No GSC velocity data found. Graph will not show health status.")
        return {}
        
    print(f"❤️ Loading health velocity from: {velocity_file.name}")
    
    try:
        with open(velocity_file, 'r', encoding='utf-8') as f:
            data = json.load(f)
            
        slug_map = {}
        for key, metrics in data.items():
            # Skip metadata keys
            if key.startswith("_"):
                continue
                
            # Extract slug if key is a URL, or use as is
            slug = key.strip('/').split('/')[-1]
            slug_map[slug] = metrics
            
        return slug_map
    except Exception as e:
        print(f"⚠️ Error loading velocity data: {e}")
        return {}

def get_cluster_label(df_cluster, market_data=None):
    """Determines the name of a Hub."""
    all_keywords = [kw for sublist in df_cluster['keywords'] for kw in sublist]
    if not all_keywords:
        return "Misc"
    
    counts = Counter(all_keywords)
    candidates = counts.most_common(5)
    
    if market_data:
        best_kw = candidates[0][0]
        best_score = -1
        
        for kw, freq in candidates:
            vol = market_data.get(kw.lower().strip(), 0)
            score = freq * np.log1p(vol)
            
            if score > best_score:
                best_score = score
                best_kw = kw
        return best_kw
        
    return candidates[0][0]

def calculate_gravity(keywords, market_data):
    """Calculates additional node radius based on max keyword volume."""
    if not market_data or not keywords:
        return 0
    
    max_vol = 0
    for kw in keywords:
        k_clean = kw.lower().strip()
        vol = market_data.get(k_clean, 0)
        if vol > max_vol:
            max_vol = vol
            
    if max_vol > 0:
        return np.log1p(max_vol)
    return 0

def recursive_cluster(df_slice, parent_id, current_depth, nodes, links, market_data, velocity_data, vectorizer=None):
    """The Gold Panning Recursive Engine."""
    df = df_slice.copy()
    
    # --- STOP CONDITION ---
    if len(df) <= TARGET_BRANCHING_FACTOR + GOLD_PAN_SIZE:
        for _, row in df.iterrows():
            # Gravity
            gravity_boost = calculate_gravity(row['keywords'], market_data)
            
            # Health/Velocity
            slug = re.sub(r'^\d{4}-\d{2}-\d{2}-', '', row['id'])
            health = velocity_data.get(slug, {})
            
            node = {
                "id": row['id'],
                "group": "article",
                "depth": current_depth,
                "label": row['label'],
                "val": 5 + gravity_boost,
                "parentId": parent_id,
                # Inject Health Data
                "status": health.get("status", "unknown"),
                "velocity": health.get("velocity", 0),
                "clicks": health.get("total_clicks", 0)
            }
            nodes.append(node)
            links.append({
                "source": parent_id,
                "target": row['id'],
                "type": "article_link"
            })
        return

    # --- VECTORIZATION & CLUSTERING ---
    if vectorizer is None:
        vectorizer = TfidfVectorizer(stop_words='english', max_features=1000)
    
    try:
        tfidf_matrix = vectorizer.fit_transform(df['soup'])
        
        n_components = min(5, len(df) - 1) 
        if n_components > 1:
            svd = TruncatedSVD(n_components=n_components)
            matrix = svd.fit_transform(tfidf_matrix)
        else:
            matrix = tfidf_matrix

        kmeans = MiniBatchKMeans(
            n_clusters=TARGET_BRANCHING_FACTOR,
            random_state=42,
            n_init=10,
            batch_size=256
        )
        clusters = kmeans.fit_predict(matrix)
        df.loc[:, 'cluster'] = clusters 
        
        # --- RECURSION ---
        for cluster_id in range(TARGET_BRANCHING_FACTOR):
            cluster_data = df[df['cluster'] == cluster_id]
            
            if len(cluster_data) == 0:
                continue
            
            hub_label = get_cluster_label(cluster_data, market_data)
            new_hub_id = f"{parent_id}_{cluster_id}"
            
            hub_base_val = max(10, 40 - (current_depth * 10))
            hub_gravity = 0
            if market_data:
                 vol = market_data.get(hub_label.lower().strip(), 0)
                 if vol > 0:
                     hub_gravity = np.log1p(vol) * 1.5
            
            nodes.append({
                "id": new_hub_id,
                "group": "hub",
                "depth": current_depth + 1,
                "label": hub_label,
                "val": hub_base_val + hub_gravity,
                "parentId": parent_id,
                "status": "hub" # Hubs are neutral
            })
            
            links.append({
                "source": parent_id,
                "target": new_hub_id,
                "type": "hub_link"
            })
            
            recursive_cluster(
                cluster_data, 
                new_hub_id, 
                current_depth + 1, 
                nodes, 
                links,
                market_data,
                velocity_data
            )
            
    except ValueError as e:
        print(f"⚠️ Clustering fallback at depth {current_depth}: {e}")
        for _, row in df.iterrows():
            gravity_boost = calculate_gravity(row['keywords'], market_data)
            slug = re.sub(r'^\d{4}-\d{2}-\d{2}-', '', row['id'])
            health = velocity_data.get(slug, {})
            
            nodes.append({
                "id": row['id'],
                "group": "article",
                "depth": current_depth,
                "label": row['label'],
                "val": 5 + gravity_boost,
                "parentId": parent_id,
                "status": health.get("status", "unknown"),
                "velocity": health.get("velocity", 0),
                "clicks": health.get("total_clicks", 0)
            })
            links.append({
                "source": parent_id,
                "target": row['id'],
                "type": "article_link"
            })

def main():
    print("🚀 Initializing Hierarchy Builder...")
    
    df = load_shards(CONTEXT_DIR)
    if df.empty:
        print("❌ No data found. Check CONTEXT_DIR path.")
        return
        
    market_data = load_market_data()
    velocity_data = load_velocity_data() # Load GSC Velocity

    nodes = [{
        "id": "hub_0",
        "group": "root",
        "depth": 0,
        "label": "HOME",
        "val": 50,
        "parentId": None,
        "status": "root"
    }]
    links = []

    print(f"🧠 Clustering {len(df)} articles using Rule of {TARGET_BRANCHING_FACTOR}...")
    recursive_cluster(df, "hub_0", 0, nodes, links, market_data, velocity_data)

    output_data = {"nodes": nodes, "links": links}
    
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(output_data, f, indent=None)
        
    print(f"✅ Hierarchy generated: {len(nodes)} nodes, {len(links)} links.")
    print(f"💾 Saved to {OUTPUT_FILE}")
    
    # Inject into HTML
    try:
        html_path = Path("ideal_hierarchy_master.html")
        if html_path.exists():
            print("💉 Injecting data into HTML visualization...")
            with open(html_path, 'r', encoding='utf-8') as f:
                content = f.read()
            
            json_str = json.dumps(output_data)
            match = re.search(r'const rawGraph = \{.*?\};', content, flags=re.DOTALL)
            
            if match:
                start, end = match.span()
                new_content = content[:start] + f'const rawGraph = {json_str};' + content[end:]
                
                with open("ideal_hierarchy_master_real.html", 'w', encoding='utf-8') as f:
                    f.write(new_content)
                print("✅ Created 'ideal_hierarchy_master_real.html' with live data.")
            else:
                print("⚠️ Could not find 'const rawGraph = {...};' placeholder in HTML file.")
                
    except Exception as e:
        print(f"⚠️ HTML Injection failed: {e}")

if __name__ == "__main__":
    main()

import json
import glob
from pathlib import Path
import pandas as pd
import numpy as np
import frontmatter  # Requires: pip install python-frontmatter
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import MiniBatchKMeans
from sklearn.decomposition import TruncatedSVD
from collections import Counter
import re
import warnings

# Silence warnings
warnings.filterwarnings('ignore', category=pd.errors.SettingWithCopyWarning)

# --- CONFIGURATION ---
# Paths relative to the script execution or absolute
CONTEXT_DIR = Path("/home/mike/repos/MikeLev.in/_posts/_context")
POSTS_DIR = Path("/home/mike/repos/MikeLev.in/_posts") 
OUTPUT_FILE = "navgraph.json"

TARGET_BRANCHING_FACTOR = 7  # Rule of 7
GOLD_PAN_SIZE = 5            # Articles to keep at this level
MIN_CLUSTER_SIZE = 5         # Minimum items to force a split

def slugify(text):
    """Turns a label into a URL-friendly slug."""
    text = text.lower()
    text = re.sub(r'[^a-z0-9\s-]', '', text)
    text = re.sub(r'\s+', '-', text)
    return text.strip('-')

def load_enriched_shards():
    """
    Ingests shards AND merges with Frontmatter from actual Markdown files.
    This ensures we have the canonical permalink and manual description.
    """
    shards = []
    files = list(CONTEXT_DIR.glob("*.json"))
    print(f"💎 Loading {len(files)} shards & enriching from Markdown...")
    
    for f in files:
        try:
            # 1. Load the AI Context (The Semantic Signal)
            with open(f, 'r', encoding='utf-8') as file:
                data = json.load(file)
            
            # 2. Load the Physical Article (The Metadata)
            md_path = POSTS_DIR / f"{f.stem}.md"
            if not md_path.exists():
                print(f"⚠️ Warning: Markdown file missing for {f.name}")
                continue

            post = frontmatter.load(md_path)
            
            # 3. Create the Semantic Soup
            soup = (
                (data.get('t', '') + " ") * 3 + 
                (" ".join(data.get('kw', [])) + " ") * 2 + 
                " ".join(data.get('sub', []))
            )

            # 4. Build the Object
            # FIX: Ensure date is a string (YAML parser might return datetime object)
            date_val = post.metadata.get('date', data.get('d', ''))
            
            shards.append({
                "id": f.stem,
                "title": post.metadata.get('title', data.get('t', 'Untitled')),
                "permalink": post.metadata.get('permalink', f"/{f.stem}/"),
                "description": post.metadata.get('description', data.get('s', '')),
                "date": str(date_val), 
                "soup": soup,
                "keywords": data.get('kw', []) + data.get('sub', [])
            })

        except Exception as e:
            print(f"⚠️ Error loading {f.name}: {e}")
            
    return pd.DataFrame(shards)

def load_market_data(directory=Path(".")):
    """Loads SEMRush/GSC CSV data for gravity weighting."""
    if not directory.exists():
        directory = Path(__file__).parent

    files = list(directory.glob("*bulk_us*.csv"))
    if not files:
        return {}
    
    latest_file = max(files, key=lambda f: f.stat().st_mtime)
    print(f"💰 Loading market gravity from: {latest_file.name}")
    
    try:
        df = pd.read_csv(latest_file)
        market_map = {}
        for _, row in df.iterrows():
            kw = str(row['Keyword']).lower().strip()
            try: vol = int(row['Volume'])
            except: vol = 0
            market_map[kw] = vol
        return market_map
    except:
        return {}

def load_velocity_data(directory=Path(".")):
    """Loads GSC velocity/health data."""
    if not directory.exists():
        directory = Path(__file__).parent
        
    velocity_file = directory / "gsc_velocity.json"
    if not velocity_file.exists():
        return {}
        
    print(f"❤️ Loading health velocity from: {velocity_file.name}")
    try:
        with open(velocity_file, 'r', encoding='utf-8') as f:
            data = json.load(f)
        
        slug_map = {}
        for key, metrics in data.items():
            # Skip metadata keys
            if key.startswith("_"):
                continue

            slug = key.strip('/').split('/')[-1]
            slug_map[slug] = metrics
            
        return slug_map
    except:
        return {}

def get_cluster_label(df_cluster, market_data):
    """Determines the best name for a Hub."""
    all_keywords = [kw for sublist in df_cluster['keywords'] for kw in sublist]
    if not all_keywords: return "Misc"
    
    counts = Counter(all_keywords)
    candidates = counts.most_common(5)
    
    best_kw = candidates[0][0]
    best_score = -1
    
    if market_data:
        for kw, freq in candidates:
            vol = market_data.get(kw.lower().strip(), 0)
            score = freq * np.log1p(vol)
            if score > best_score:
                best_score = score
                best_kw = kw
    
    return best_kw

def calculate_gravity(row, market_data, velocity_data):
    """Calculates the sorting score."""
    max_vol = 0
    if market_data:
        for kw in row['keywords']:
            vol = market_data.get(kw.lower().strip(), 0)
            if vol > max_vol: max_vol = vol
            
    # Match Logic for GSC (stripping date prefix usually found in filenames)
    # Adjust this regex if your filenames don't start with YYYY-MM-DD
    slug_match = re.search(r'\d{4}-\d{2}-\d{2}-(.*)', row['id'])
    slug = slug_match.group(1) if slug_match else row['id']
    
    gsc_clicks = 0
    if velocity_data:
        gsc_clicks = velocity_data.get(slug, {}).get('total_clicks', 0)

    # Composite Score
    # FIX: Cast to native float for JSON serialization
    return float((np.log1p(max_vol) * 1.0) + (np.log1p(gsc_clicks) * 5.0))

def build_tree_recursive(df_slice, current_depth, market_data, velocity_data, vectorizer=None, used_slugs=None):
    """
    Recursively builds the NavGraph dictionary.
    """
    if used_slugs is None: used_slugs = set()
    
    # 1. Score and Sort
    df = df_slice.copy()
    df['score'] = df.apply(lambda row: calculate_gravity(row, market_data, velocity_data), axis=1)
    df = df.sort_values('score', ascending=False)
    
    node = {
        "children_hubs": [],
        "children_articles": []
    }
    
    # 2. Stop Condition / Leaf Node
    if len(df) <= TARGET_BRANCHING_FACTOR + GOLD_PAN_SIZE:
        # Dump everything as articles
        for _, row in df.iterrows():
            node["children_articles"].append({
                "title": row['title'],
                "permalink": row['permalink'],
                "blurb": row['description'],
                "date": row['date'],
                "gravity": row['score']
            })
        return node

    # 3. Gold Pan (Top Articles stay here)
    gold_df = df.head(GOLD_PAN_SIZE)
    for _, row in gold_df.iterrows():
        node["children_articles"].append({
            "title": row['title'],
            "permalink": row['permalink'],
            "blurb": row['description'],
            "date": row['date'],
            "gravity": row['score']
        })

    # 4. Cluster the Remainder
    remainder_df = df.iloc[GOLD_PAN_SIZE:].copy()
    
    if vectorizer is None:
        vectorizer = TfidfVectorizer(stop_words='english', max_features=1000)
    
    try:
        tfidf_matrix = vectorizer.fit_transform(remainder_df['soup'])
        n_components = min(5, len(remainder_df) - 1)
        matrix = TruncatedSVD(n_components).fit_transform(tfidf_matrix) if n_components > 1 else tfidf_matrix
        
        kmeans = MiniBatchKMeans(n_clusters=TARGET_BRANCHING_FACTOR, random_state=42, n_init=10)
        clusters = kmeans.fit_predict(matrix)
        remainder_df.loc[:, 'cluster'] = clusters
        
        for cluster_id in range(TARGET_BRANCHING_FACTOR):
            cluster_data = remainder_df[remainder_df['cluster'] == cluster_id]
            if len(cluster_data) == 0: continue
            
            # Determine Hub Identity
            hub_label = get_cluster_label(cluster_data, market_data)
            
            # Slug Deduplication Strategy
            base_slug = slugify(hub_label)
            slug = base_slug
            counter = 1
            while slug in used_slugs:
                counter += 1
                slug = f"{base_slug}-{counter}"
            used_slugs.add(slug)
            
            # Recursive Call
            child_node = build_tree_recursive(cluster_data, current_depth + 1, market_data, velocity_data, vectorizer, used_slugs)
            
            # Enrich Child Node with Hub Metadata
            child_node["title"] = hub_label
            child_node["permalink"] = f"/{slug}/"
            child_node["id"] = f"hub_{slug}"
            
            node["children_hubs"].append(child_node)
            
    except Exception as e:
        print(f"⚠️ Clustering failed at depth {current_depth}: {e}. Dumping as flat articles.")
        for _, row in remainder_df.iterrows():
             node["children_articles"].append({
                "title": row['title'],
                "permalink": row['permalink'],
                "gravity": row['score']
            })

    return node

def main():
    print("🚀 Initializing NavGraph Builder...")
    
    df = load_enriched_shards()
    if df.empty:
        print("❌ No data. Check paths.")
        return
        
    market_data = load_market_data()
    velocity_data = load_velocity_data()
    
    print(f"🧠 Building NavGraph from {len(df)} articles...")
    
    # Build the Root
    nav_tree = build_tree_recursive(df, 0, market_data, velocity_data)
    
    # Decorate Root
    nav_tree["title"] = "Home"
    nav_tree["permalink"] = "/"
    nav_tree["id"] = "root"
    
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(nav_tree, f, indent=2)
        
    print(f"✅ NavGraph generated.")
    print(f"💾 Saved to {OUTPUT_FILE}")
    print("👉 Next Step: Run the Jekyll Page Generator against this JSON.")

if __name__ == "__main__":
    main()