ResearchAgent/src/scout_logic.py

"""
Scout-Logik: Domain scannen, Links extrahieren, via OpenRouter Publikations-URL identifizieren.
"""
import json
import re
from typing import Any
from urllib.parse import urlparse

import httpx
from playwright.async_api import async_playwright

# OpenRouter Base-URL und Modell
OPENROUTER_BASE = "https://openrouter.ai/api/v1"
DEFAULT_MODEL = "google/gemini-flash-1.5-8b"


async def _fetch_links_with_playwright(domain: str) -> list[dict[str, str]]:
    """
    Lädt die Startseite der Domain mit Playwright (headless) und
    extrahiert alle <a>-Tags (Text und Href).
    """
    # Domain mit Schema normalisieren
    url = domain if domain.startswith("http") else f"https://{domain}"
    links: list[dict[str, str]] = []

    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        try:
            page = await browser.new_page()
            await page.goto(url, wait_until="domcontentloaded", timeout=15000)
            # Alle <a href="..."> auslesen
            links = await page.evaluate(
                """() => {
                const anchors = document.querySelectorAll('a[href]');
                return Array.from(anchors).map(a => ({
                    text: (a.textContent || '').trim().slice(0, 200),
                    href: a.getAttribute('href') || ''
                })).filter(x => x.href);
            }"""
            )
        finally:
            await browser.close()

    return links


def _make_absolute(href: str, base_url: str) -> str:
    """Macht relative URLs absolut (einfache Heuristik)."""
    if not href or href.startswith("#"):
        return ""
    if href.startswith("http://") or href.startswith("https://"):
        return href
    base = base_url.rstrip("/")
    if href.startswith("/"):
        parsed = urlparse(base)
        return f"{parsed.scheme}://{parsed.netloc}{href}"
    return f"{base}/{href}"


async def _ask_openrouter(api_key: str, links: list[dict[str, str]], domain: str) -> str | None:
    """
    Sendet die Link-Liste an OpenRouter und fordert die beste Publikations-URL an.
    Erwartet Antwort im Format: {"url": "..."}
    """
    base_url = domain if domain.startswith("http") else f"https://{domain}"
    prompt = (
        "Analysiere diese Links einer Unternehmensberatung. "
        "Welcher Link führt zur Seite mit Reports, Insights oder Fachartikeln? "
        "Antworte NUR mit der absoluten URL im JSON-Format: {'url': '...'}"
    )
    links_text = "\n".join(
        f"- {l.get('text', '')} -> {_make_absolute(l.get('href', ''), base_url)}"
        for l in links[:80]  # Begrenzen, um Token-Limit zu schonen
    )
    user_content = f"Links von {domain}:\n{links_text}"

    async with httpx.AsyncClient(timeout=30.0) as client:
        resp = await client.post(
            f"{OPENROUTER_BASE}/chat/completions",
            headers={
                "Authorization": f"Bearer {api_key}",
                "Content-Type": "application/json",
                "HTTP-Referer": base_url,
            },
            json={
                "model": DEFAULT_MODEL,
                "messages": [
                    {"role": "system", "content": prompt},
                    {"role": "user", "content": user_content},
                ],
                "max_tokens": 256,
            },
        )
        resp.raise_for_status()
        data = resp.json()
        choice = (data.get("choices") or [None])[0]
        if not choice:
            return None
        content = (choice.get("message") or {}).get("content") or ""
        if not content.strip():
            return None

    # JSON aus Antwort extrahieren (falls von Markdown umgeben)
    content = content.strip()
    json_match = re.search(r"\{[^{}]*\"url\"[^{}]*\}", content)
    if json_match:
        try:
            obj = json.loads(json_match.group())
            return (obj.get("url") or "").strip() or None
        except json.JSONDecodeError:
            pass
    try:
        obj = json.loads(content)
        return (obj.get("url") or "").strip() or None
    except json.JSONDecodeError:
        return None


async def get_publication_url(domain: str, *, api_key: str | None = None) -> dict[str, Any]:
    """
    Hauptfunktion: Domain scannen, Links an OpenRouter senden,
    gefundene Publikations-URL zurückgeben.
    """
    import os
    key = api_key or os.getenv("OPENROUTER_API_KEY")
    if not key:
        return {"url": None, "error": "OPENROUTER_API_KEY nicht gesetzt"}

    try:
        links = await _fetch_links_with_playwright(domain)
    except Exception as e:
        return {"url": None, "error": f"Playwright/Scrape-Fehler: {e!s}"}

    if not links:
        return {"url": None, "error": "Keine Links auf der Seite gefunden"}

    try:
        url = await _ask_openrouter(key, links, domain)
        return {"url": url, "error": None}
    except Exception as e:
        return {"url": None, "error": f"OpenRouter-Fehler: {e!s}"}