"""
Scout-Logik: Domain scannen, Links extrahieren, via OpenRouter Publikations-URL identifizieren.
"""
import json
import re
from typing import Any
from urllib.parse import urlparse
import httpx
from playwright.async_api import async_playwright
# OpenRouter Base-URL und Modell
OPENROUTER_BASE = "https://openrouter.ai/api/v1"
DEFAULT_MODEL = "google/gemini-flash-1.5-8b"
async def _fetch_links_with_playwright(domain: str) -> list[dict[str, str]]:
"""
Lädt die Startseite der Domain mit Playwright (headless) und
extrahiert alle -Tags (Text und Href).
"""
# Domain mit Schema normalisieren
url = domain if domain.startswith("http") else f"https://{domain}"
links: list[dict[str, str]] = []
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
try:
page = await browser.new_page()
await page.goto(url, wait_until="domcontentloaded", timeout=15000)
# Alle auslesen
links = await page.evaluate(
"""() => {
const anchors = document.querySelectorAll('a[href]');
return Array.from(anchors).map(a => ({
text: (a.textContent || '').trim().slice(0, 200),
href: a.getAttribute('href') || ''
})).filter(x => x.href);
}"""
)
finally:
await browser.close()
return links
def _make_absolute(href: str, base_url: str) -> str:
"""Macht relative URLs absolut (einfache Heuristik)."""
if not href or href.startswith("#"):
return ""
if href.startswith("http://") or href.startswith("https://"):
return href
base = base_url.rstrip("/")
if href.startswith("/"):
parsed = urlparse(base)
return f"{parsed.scheme}://{parsed.netloc}{href}"
return f"{base}/{href}"
async def _ask_openrouter(api_key: str, links: list[dict[str, str]], domain: str) -> str | None:
"""
Sendet die Link-Liste an OpenRouter und fordert die beste Publikations-URL an.
Erwartet Antwort im Format: {"url": "..."}
"""
base_url = domain if domain.startswith("http") else f"https://{domain}"
prompt = (
"Analysiere diese Links einer Unternehmensberatung. "
"Welcher Link führt zur Seite mit Reports, Insights oder Fachartikeln? "
"Antworte NUR mit der absoluten URL im JSON-Format: {'url': '...'}"
)
links_text = "\n".join(
f"- {l.get('text', '')} -> {_make_absolute(l.get('href', ''), base_url)}"
for l in links[:80] # Begrenzen, um Token-Limit zu schonen
)
user_content = f"Links von {domain}:\n{links_text}"
async with httpx.AsyncClient(timeout=30.0) as client:
resp = await client.post(
f"{OPENROUTER_BASE}/chat/completions",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"HTTP-Referer": base_url,
},
json={
"model": DEFAULT_MODEL,
"messages": [
{"role": "system", "content": prompt},
{"role": "user", "content": user_content},
],
"max_tokens": 256,
},
)
resp.raise_for_status()
data = resp.json()
choice = (data.get("choices") or [None])[0]
if not choice:
return None
content = (choice.get("message") or {}).get("content") or ""
if not content.strip():
return None
# JSON aus Antwort extrahieren (falls von Markdown umgeben)
content = content.strip()
json_match = re.search(r"\{[^{}]*\"url\"[^{}]*\}", content)
if json_match:
try:
obj = json.loads(json_match.group())
return (obj.get("url") or "").strip() or None
except json.JSONDecodeError:
pass
try:
obj = json.loads(content)
return (obj.get("url") or "").strip() or None
except json.JSONDecodeError:
return None
async def get_publication_url(domain: str, *, api_key: str | None = None) -> dict[str, Any]:
"""
Hauptfunktion: Domain scannen, Links an OpenRouter senden,
gefundene Publikations-URL zurückgeben.
"""
import os
key = api_key or os.getenv("OPENROUTER_API_KEY")
if not key:
return {"url": None, "error": "OPENROUTER_API_KEY nicht gesetzt"}
try:
links = await _fetch_links_with_playwright(domain)
except Exception as e:
return {"url": None, "error": f"Playwright/Scrape-Fehler: {e!s}"}
if not links:
return {"url": None, "error": "Keine Links auf der Seite gefunden"}
try:
url = await _ask_openrouter(key, links, domain)
return {"url": url, "error": None}
except Exception as e:
return {"url": None, "error": f"OpenRouter-Fehler: {e!s}"}