Klappe Geizhals-Filtergruppen beim Lernen auf und nimm neue Kategorien ueber den Quellpfad auf.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/api/discovery.py

@@ -11,6 +11,7 @@ from app.api.schemas import (
     DiscoverySearchPatch,
     SourceAttributeMappingIn,
     SourceAttributeMappingPatch,
+    SourceCategoryIn,
 )
 from app.persistence.db import get_session
 from app.services.catalog_service import CatalogService
@@ -30,6 +31,11 @@ def list_source_categories(source_id: str, session: Session = Depends(get_sessio
     return {"items": CatalogService(session).source_categories(source_id)}
 
 
+@router.post("/sources/{source_id}/categories", status_code=201)
+def register_source_category(source_id: str, payload: SourceCategoryIn, session: Session = Depends(get_session)):
+    return CatalogService(session).register_source_category(source_id, payload.model_dump())
+
+
 @router.post("/sources/{source_id}/categories/{category}/discover-taxonomy", status_code=201)
 def discover_taxonomy(source_id: str, category: str, session: Session = Depends(get_session)):
     return TaxonomyService(session).discover(source_id, category)

backend/app/api/schemas.py

@@ -189,6 +189,13 @@ class ProjectCriteriaIn(StrictModel):
     items: list[dict[str, Any]]
 
 
+class SourceCategoryIn(StrictModel):
+    source_url: str
+    name: str | None = None
+    product_category_id: str | None = None
+    product_category_name: str | None = None
+
+
 class CriterionDefinitionIn(StrictModel):
     key: str
     value_type: str

backend/app/services/catalog_service.py

@@ -67,6 +67,33 @@ class CatalogService:
             for row in rows
         ]
 
+    def register_source_category(self, source_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+        if self.catalog.get_source(source_id) is None:
+            raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
+        from app.sources.geizhals.category_url import parse_category_key, slugify
+        from app.services.discovery_service import TaxonomyService
+
+        source_url = str(payload["source_url"]).strip()
+        category_key = parse_category_key(source_url)
+        name = str(payload.get("name") or category_key)
+        product_category_id = str(payload.get("product_category_id") or slugify(name))
+        ensure_product_category(self.session, product_category_id, str(payload.get("product_category_name") or name))
+        category = ensure_source_category(
+            self.session,
+            source_id=source_id,
+            external_category_key=category_key,
+            product_category_id=product_category_id,
+            name=name,
+            source_url=source_url,
+        )
+        category.source_url = source_url
+        category.name = name
+        category.product_category_id = product_category_id
+        self.session.flush()
+        TaxonomyService(self.session).discover(source_id, category_key)
+        items = self.source_categories(source_id)
+        return next(item for item in items if item["external_category_key"] == category_key)
+
     def source_categories(self, source_id: str) -> list[dict[str, Any]]:
         if self.catalog.get_source(source_id) is None:
             raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})

backend/app/services/discovery_service.py

@@ -100,7 +100,18 @@ class TaxonomyService:
         if catalog.get_source(source_id) is None:
             raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
         adapter = self.adapter or discovery_adapter(source_id)
+        existing = self.session.scalars(
+            select(SourceCategoryRecord).where(
+                SourceCategoryRecord.source_id == source_id,
+                SourceCategoryRecord.external_category_key == category_key,
+            )
+        ).first()
         try:
+            payload = adapter.discover_taxonomy(
+                category_key,
+                source_url=None if existing is None else existing.source_url,
+            )
+        except TypeError:
             payload = adapter.discover_taxonomy(category_key)
         except AppError as exc:
             category = self.session.scalars(

backend/app/sources/geizhals/adapter.py

@@ -8,6 +8,7 @@ from urllib.parse import parse_qs, urlparse
 from app.discovery.enums import FilterType
 from app.errors import AppError
 from app.settings import get_settings
+from app.sources.geizhals.category_url import parse_category_key, with_extended_filters
 from app.sources.geizhals.live_page import load_category_page, load_result_page
 from app.sources.geizhals.result_parser import parse_result_html, with_page
 from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
@@ -23,16 +24,17 @@ class GeizhalsAdapter:
         settings = get_settings()
         self.live = (not settings.geizhals_use_fixtures) if live is None else live
 
-    def discover_taxonomy(self, category_ref: str) -> dict[str, Any]:
+    def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]:
         if not self.live:
             return self._load(f"{category_ref}-taxonomy.json")
-        url = f"https://geizhals.de/?cat={category_ref}&promode=true"
+        url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}")
+        category_ref = parse_category_key(url)
         html, mode, status = load_category_page(
             url,
             category_ref=category_ref,
             playwright=get_settings().geizhals_playwright,
         )
-        payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=url)
+        payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url)
         if payload.get("blocked") or not payload.get("filters"):
             raise AppError(
                 422,
@@ -48,7 +50,8 @@ class GeizhalsAdapter:
     def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
         from app.discovery.urlgen import generate_search_url
 
-        category_url = f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
+        category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None)
+        category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
         url, internal_only = generate_search_url(
             category_url=category_url,
             criteria_items=criteria,

backend/app/sources/geizhals/category_url.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
+
+from app.errors import AppError
+
+CATEGORY_PATH_RE = re.compile(r"/cat/([^/?#]+)", re.IGNORECASE)
+
+
+def parse_category_key(source_url: str) -> str:
+    parts = urlsplit(source_url.strip())
+    query = parse_qs(parts.query)
+    if query.get("cat"):
+        return str(query["cat"][0])
+    match = CATEGORY_PATH_RE.search(parts.path)
+    if match:
+        return match.group(1)
+    raise AppError(422, "SOURCE_URL_INVALID", "Quellpfad muss eine Geizhals-Kategorie enthalten (cat=…)")
+
+
+def slugify(value: str) -> str:
+    text = value.strip().casefold().translate(str.maketrans("äöüß", "aous"))
+    text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
+    return text or "category"
+
+
+def with_extended_filters(source_url: str) -> str:
+    parts = urlsplit(source_url.strip())
+    query = [(name, value) for name, value in _query_items(parts.query) if name != "promode"]
+    query.append(("promode", "true"))
+    return urlunsplit((parts.scheme or "https", parts.netloc, parts.path or "/", urlencode(query), parts.fragment))
+
+
+def _query_items(query: str) -> list[tuple[str, str]]:
+    from urllib.parse import parse_qsl
+
+    return parse_qsl(query, keep_blank_values=True)

backend/app/sources/geizhals/live_page.py

@@ -15,6 +15,8 @@ BROWSER_HEADERS = {
     "Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
 }
 EXPAND_LABELS = (
+    "Erweiterte Filter",
+    "Alle Filter ausklappen",
     "Immer alle Filter anzeigen",
     "weitere Filter anzeigen",
 )
@@ -42,19 +44,19 @@ def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, in
 
 def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]:
     status, html = 0, ""
+    if playwright:
+        try:
+            status, html = _fetch_playwright(url, expand_filters=True)
+            if not _blocked(status, html):
+                return html, "live_playwright", status
+        except Exception:
+            pass
     try:
         status, html = _fetch_http(url)
         if not _blocked(status, html):
             return html, "live_http", status
     except Exception:
-        status, html = 0, ""
-    if playwright:
-        try:
-            status, html = _fetch_playwright(url)
-            if not _blocked(status, html):
-                return html, "live_playwright", status
-        except Exception:
-            pass
+        status, html = status or 0, html or ""
     recorded = FIXTURE_DIR / f"{category_ref}-category.html"
     if recorded.exists():
         return recorded.read_text(encoding="utf-8"), "recorded_html", 0
@@ -73,7 +75,7 @@ def _fetch_http(url: str) -> tuple[int, str]:
     return response.status_code, response.text
 
 
-def _fetch_playwright(url: str) -> tuple[int, str]:
+def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, str]:
     try:
         from playwright.sync_api import sync_playwright
     except ImportError as exc:
@@ -87,17 +89,36 @@ def _fetch_playwright(url: str) -> tuple[int, str]:
         page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS)
         response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
         page.wait_for_timeout(4000)
+        if expand_filters:
+            _expand_filter_sections(page)
         for label in EXPAND_LABELS:
             button = page.get_by_text(label, exact=False)
             if button.count():
-                button.first.click(timeout=5000)
-                page.wait_for_timeout(1500)
+                try:
+                    button.first.click(timeout=4000)
+                    page.wait_for_timeout(800)
+                except Exception:
+                    continue
+        if expand_filters:
+            _expand_filter_sections(page)
         html = page.content()
         status = response.status if response is not None else 200
         browser.close()
         return status, html
 
 
+def _expand_filter_sections(page: object) -> None:
+    locator = page.locator('button[aria-label*="aufklappen"]')
+    for _ in range(40):
+        if locator.count() == 0:
+            return
+        try:
+            locator.first.click(timeout=3000)
+            page.wait_for_timeout(700)
+        except Exception:
+            return
+
+
 def _blocked(status: int, html: str) -> bool:
     if status in {401, 403, 429}:
         return True

backend/tests/unit/test_geizhals_category_url.py

@@ -0,0 +1,18 @@
+from app.errors import AppError
+from app.sources.geizhals.category_url import parse_category_key, slugify, with_extended_filters
+import pytest
+
+
+def test_parse_category_key_from_query() -> None:
+    assert parse_category_key("https://geizhals.de/?cat=gra16_512&xf=1_AMD") == "gra16_512"
+
+
+def test_parse_category_key_rejects_unknown_url() -> None:
+    with pytest.raises(AppError) as exc:
+        parse_category_key("https://geizhals.de/")
+    assert exc.value.code == "SOURCE_URL_INVALID"
+
+
+def test_slugify_and_extended_filters() -> None:
+    assert slugify("Grafikkarten") == "grafikkarten"
+    assert "promode=true" in with_extended_filters("https://geizhals.de/?cat=nb")

frontend/src/api.ts

@@ -41,6 +41,8 @@ export const api = {
   sourceCategories: (source: string) => request<{ items: SourceCategory[] }>(`/sources/${source}/categories`),
   discoverTaxonomy: (source: string, category: string) =>
     request<unknown>(`/sources/${source}/categories/${category}/discover-taxonomy`, { method: "POST" }),
+  createSourceCategory: (source: string, payload: { source_url: string; name?: string }) =>
+    request<SourceCategory>(`/sources/${source}/categories`, { method: "POST", body: JSON.stringify(payload) }),
   filters: (source: string, category: string) => request<{ filters: SourceFilter[] }>(`/sources/${source}/categories/${category}/filters`),
 };
 
@@ -166,5 +168,6 @@ export type SourceCategory = {
   status: string;
   taxonomy_status?: string;
   product_category_id?: string | null;
+  source_url?: string | null;
 };
 export type SourceFilter = { label: string; external_filter_key: string; filter_type: string; status: string };

frontend/src/pages/AdminPage.tsx

@@ -1,4 +1,5 @@
 import { useMutation, useQuery } from "@tanstack/react-query";
+import { useState } from "react";
 import { api } from "../api";
 import { EmptyState, LoadingState } from "../components/State";
 
@@ -10,36 +11,79 @@ export function AdminPage() {
     queryFn: () => api.sourceCategories(source),
     enabled: Boolean(source),
   });
-  const category = categories.data?.items[0]?.external_category_key || "";
+  const [category, setCategory] = useState("");
+  const selectedKey = category || categories.data?.items[0]?.external_category_key || "";
   const filters = useQuery({
-    queryKey: ["filters", source, category],
-    queryFn: () => api.filters(source, category),
-    enabled: Boolean(source && category),
+    queryKey: ["filters", source, selectedKey],
+    queryFn: () => api.filters(source, selectedKey),
+    enabled: Boolean(source && selectedKey),
   });
+  const [sourceUrl, setSourceUrl] = useState("");
+  const [name, setName] = useState("");
   const discover = useMutation({
-    mutationFn: () => api.discoverTaxonomy(source, category),
+    mutationFn: () => api.discoverTaxonomy(source, selectedKey),
     onSuccess: () => {
       categories.refetch();
       filters.refetch();
     },
   });
+  const register = useMutation({
+    mutationFn: () => api.createSourceCategory(source, { source_url: sourceUrl, name: name || undefined }),
+    onSuccess: (created) => {
+      setCategory(created.external_category_key);
+      setSourceUrl("");
+      setName("");
+      categories.refetch();
+    },
+  });
   if (sources.isLoading) return <LoadingState />;
-  const selected = categories.data?.items.find((item) => item.external_category_key === category);
+  const selected = categories.data?.items.find((item) => item.external_category_key === selectedKey);
   return (
     <section className="grid">
       <h1>Quellen und Taxonomie</h1>
-      <p className="muted">Technische IDs sind nur hier sichtbar.</p>
+      <p className="muted">
+        Weitere Produktkategorien kommen über den Geizhals-Quellpfad. Beim Lernen werden zugeklappte Filtergruppen
+        geöffnet; das erste HTML enthält nur die Sektion „Allgemein“.
+      </p>
+      <div className="card grid">
+        <label>
+          Quellpfad
+          <input
+            value={sourceUrl}
+            onChange={(event) => setSourceUrl(event.target.value)}
+            placeholder="https://geizhals.de/?cat=nb"
+          />
+        </label>
+        <label>
+          Name
+          <input value={name} onChange={(event) => setName(event.target.value)} placeholder="Notebooks" />
+        </label>
+        {register.isError ? <p className="danger">{(register.error as Error).message}</p> : null}
+        <button className="button" disabled={!source || !sourceUrl || register.isPending} onClick={() => register.mutate()}>
+          {register.isPending ? "Legt an und lernt …" : "Kategorie aus Quellpfad anlegen"}
+        </button>
+      </div>
       <div className="card grid">
         <p>Source: {source || "–"}</p>
-        <p>Kategorie: {category || "–"}</p>
+        <label>
+          Source-Kategorie
+          <select value={selectedKey} onChange={(event) => setCategory(event.target.value)}>
+            {(categories.data?.items || []).map((item) => (
+              <option key={item.external_category_key} value={item.external_category_key}>
+                {item.name} ({item.external_category_key})
+              </option>
+            ))}
+          </select>
+        </label>
+        <p>Quellpfad: {selected?.source_url || "–"}</p>
         <p>Status: {selected?.taxonomy_status || "NOT_LEARNED"}</p>
         <p>Gelernte Filter: {filters.data?.filters.length ?? 0}</p>
         {discover.isError ? <p className="danger">{(discover.error as Error).message}</p> : null}
-        <button className="button" disabled={!source || !category || discover.isPending} onClick={() => discover.mutate()}>
+        <button className="button" disabled={!source || !selectedKey || discover.isPending} onClick={() => discover.mutate()}>
           {discover.isPending ? "Lernt Taxonomie …" : "Taxonomie neu lernen"}
         </button>
       </div>
-      {!category ? (
+      {!selectedKey ? (
         <EmptyState>Keine Source-Kategorie hinterlegt.</EmptyState>
       ) : !filters.data?.filters.length ? (
         <EmptyState>Taxonomie wurde noch nicht geladen.</EmptyState>