""" Scout-Logik: Domain scannen, Links extrahieren, via OpenRouter Publikations-URL identifizieren. """ import json import re from typing import Any from urllib.parse import urlparse import httpx from playwright.async_api import async_playwright # OpenRouter Base-URL und Modell OPENROUTER_BASE = "https://openrouter.ai/api/v1" DEFAULT_MODEL = "google/gemini-flash-1.5-8b" async def _fetch_links_with_playwright(domain: str) -> list[dict[str, str]]: """ Lädt die Startseite der Domain mit Playwright (headless) und extrahiert alle -Tags (Text und Href). """ # Domain mit Schema normalisieren url = domain if domain.startswith("http") else f"https://{domain}" links: list[dict[str, str]] = [] async with async_playwright() as p: browser = await p.chromium.launch(headless=True) try: page = await browser.new_page() await page.goto(url, wait_until="domcontentloaded", timeout=15000) # Alle auslesen links = await page.evaluate( """() => { const anchors = document.querySelectorAll('a[href]'); return Array.from(anchors).map(a => ({ text: (a.textContent || '').trim().slice(0, 200), href: a.getAttribute('href') || '' })).filter(x => x.href); }""" ) finally: await browser.close() return links def _make_absolute(href: str, base_url: str) -> str: """Macht relative URLs absolut (einfache Heuristik).""" if not href or href.startswith("#"): return "" if href.startswith("http://") or href.startswith("https://"): return href base = base_url.rstrip("/") if href.startswith("/"): parsed = urlparse(base) return f"{parsed.scheme}://{parsed.netloc}{href}" return f"{base}/{href}" async def _ask_openrouter(api_key: str, links: list[dict[str, str]], domain: str) -> str | None: """ Sendet die Link-Liste an OpenRouter und fordert die beste Publikations-URL an. Erwartet Antwort im Format: {"url": "..."} """ base_url = domain if domain.startswith("http") else f"https://{domain}" prompt = ( "Analysiere diese Links einer Unternehmensberatung. " "Welcher Link führt zur Seite mit Reports, Insights oder Fachartikeln? " "Antworte NUR mit der absoluten URL im JSON-Format: {'url': '...'}" ) links_text = "\n".join( f"- {l.get('text', '')} -> {_make_absolute(l.get('href', ''), base_url)}" for l in links[:80] # Begrenzen, um Token-Limit zu schonen ) user_content = f"Links von {domain}:\n{links_text}" async with httpx.AsyncClient(timeout=30.0) as client: resp = await client.post( f"{OPENROUTER_BASE}/chat/completions", headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", "HTTP-Referer": base_url, }, json={ "model": DEFAULT_MODEL, "messages": [ {"role": "system", "content": prompt}, {"role": "user", "content": user_content}, ], "max_tokens": 256, }, ) resp.raise_for_status() data = resp.json() choice = (data.get("choices") or [None])[0] if not choice: return None content = (choice.get("message") or {}).get("content") or "" if not content.strip(): return None # JSON aus Antwort extrahieren (falls von Markdown umgeben) content = content.strip() json_match = re.search(r"\{[^{}]*\"url\"[^{}]*\}", content) if json_match: try: obj = json.loads(json_match.group()) return (obj.get("url") or "").strip() or None except json.JSONDecodeError: pass try: obj = json.loads(content) return (obj.get("url") or "").strip() or None except json.JSONDecodeError: return None async def get_publication_url(domain: str, *, api_key: str | None = None) -> dict[str, Any]: """ Hauptfunktion: Domain scannen, Links an OpenRouter senden, gefundene Publikations-URL zurückgeben. """ import os key = api_key or os.getenv("OPENROUTER_API_KEY") if not key: return {"url": None, "error": "OPENROUTER_API_KEY nicht gesetzt"} try: links = await _fetch_links_with_playwright(domain) except Exception as e: return {"url": None, "error": f"Playwright/Scrape-Fehler: {e!s}"} if not links: return {"url": None, "error": "Keine Links auf der Seite gefunden"} try: url = await _ask_openrouter(key, links, domain) return {"url": url, "error": None} except Exception as e: return {"url": None, "error": f"OpenRouter-Fehler: {e!s}"}