diff --git a/backend/app/api/discovery.py b/backend/app/api/discovery.py index c2eb574..a27ca71 100644 --- a/backend/app/api/discovery.py +++ b/backend/app/api/discovery.py @@ -11,6 +11,7 @@ from app.api.schemas import ( DiscoverySearchPatch, SourceAttributeMappingIn, SourceAttributeMappingPatch, + SourceCategoryIn, ) from app.persistence.db import get_session from app.services.catalog_service import CatalogService @@ -30,6 +31,11 @@ def list_source_categories(source_id: str, session: Session = Depends(get_sessio return {"items": CatalogService(session).source_categories(source_id)} +@router.post("/sources/{source_id}/categories", status_code=201) +def register_source_category(source_id: str, payload: SourceCategoryIn, session: Session = Depends(get_session)): + return CatalogService(session).register_source_category(source_id, payload.model_dump()) + + @router.post("/sources/{source_id}/categories/{category}/discover-taxonomy", status_code=201) def discover_taxonomy(source_id: str, category: str, session: Session = Depends(get_session)): return TaxonomyService(session).discover(source_id, category) diff --git a/backend/app/api/schemas.py b/backend/app/api/schemas.py index 1fae67d..acedebd 100644 --- a/backend/app/api/schemas.py +++ b/backend/app/api/schemas.py @@ -189,6 +189,13 @@ class ProjectCriteriaIn(StrictModel): items: list[dict[str, Any]] +class SourceCategoryIn(StrictModel): + source_url: str + name: str | None = None + product_category_id: str | None = None + product_category_name: str | None = None + + class CriterionDefinitionIn(StrictModel): key: str value_type: str diff --git a/backend/app/services/catalog_service.py b/backend/app/services/catalog_service.py index 27d519c..8c6e3f6 100644 --- a/backend/app/services/catalog_service.py +++ b/backend/app/services/catalog_service.py @@ -67,6 +67,33 @@ class CatalogService: for row in rows ] + def register_source_category(self, source_id: str, payload: dict[str, Any]) -> dict[str, Any]: + if self.catalog.get_source(source_id) is None: + raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) + from app.sources.geizhals.category_url import parse_category_key, slugify + from app.services.discovery_service import TaxonomyService + + source_url = str(payload["source_url"]).strip() + category_key = parse_category_key(source_url) + name = str(payload.get("name") or category_key) + product_category_id = str(payload.get("product_category_id") or slugify(name)) + ensure_product_category(self.session, product_category_id, str(payload.get("product_category_name") or name)) + category = ensure_source_category( + self.session, + source_id=source_id, + external_category_key=category_key, + product_category_id=product_category_id, + name=name, + source_url=source_url, + ) + category.source_url = source_url + category.name = name + category.product_category_id = product_category_id + self.session.flush() + TaxonomyService(self.session).discover(source_id, category_key) + items = self.source_categories(source_id) + return next(item for item in items if item["external_category_key"] == category_key) + def source_categories(self, source_id: str) -> list[dict[str, Any]]: if self.catalog.get_source(source_id) is None: raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) diff --git a/backend/app/services/discovery_service.py b/backend/app/services/discovery_service.py index e81dea3..1f2c16d 100644 --- a/backend/app/services/discovery_service.py +++ b/backend/app/services/discovery_service.py @@ -100,7 +100,18 @@ class TaxonomyService: if catalog.get_source(source_id) is None: raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) adapter = self.adapter or discovery_adapter(source_id) + existing = self.session.scalars( + select(SourceCategoryRecord).where( + SourceCategoryRecord.source_id == source_id, + SourceCategoryRecord.external_category_key == category_key, + ) + ).first() try: + payload = adapter.discover_taxonomy( + category_key, + source_url=None if existing is None else existing.source_url, + ) + except TypeError: payload = adapter.discover_taxonomy(category_key) except AppError as exc: category = self.session.scalars( diff --git a/backend/app/sources/geizhals/adapter.py b/backend/app/sources/geizhals/adapter.py index b140afd..682b9a9 100644 --- a/backend/app/sources/geizhals/adapter.py +++ b/backend/app/sources/geizhals/adapter.py @@ -8,6 +8,7 @@ from urllib.parse import parse_qs, urlparse from app.discovery.enums import FilterType from app.errors import AppError from app.settings import get_settings +from app.sources.geizhals.category_url import parse_category_key, with_extended_filters from app.sources.geizhals.live_page import load_category_page, load_result_page from app.sources.geizhals.result_parser import parse_result_html, with_page from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html @@ -23,16 +24,17 @@ class GeizhalsAdapter: settings = get_settings() self.live = (not settings.geizhals_use_fixtures) if live is None else live - def discover_taxonomy(self, category_ref: str) -> dict[str, Any]: + def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]: if not self.live: return self._load(f"{category_ref}-taxonomy.json") - url = f"https://geizhals.de/?cat={category_ref}&promode=true" + url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}") + category_ref = parse_category_key(url) html, mode, status = load_category_page( url, category_ref=category_ref, playwright=get_settings().geizhals_playwright, ) - payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=url) + payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url) if payload.get("blocked") or not payload.get("filters"): raise AppError( 422, @@ -48,7 +50,8 @@ class GeizhalsAdapter: def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: from app.discovery.urlgen import generate_search_url - category_url = f"https://geizhals.de/?cat={self._category(mappings, criteria)}" + category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None) + category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}" url, internal_only = generate_search_url( category_url=category_url, criteria_items=criteria, diff --git a/backend/app/sources/geizhals/category_url.py b/backend/app/sources/geizhals/category_url.py new file mode 100644 index 0000000..e9f8b82 --- /dev/null +++ b/backend/app/sources/geizhals/category_url.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import re +from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit + +from app.errors import AppError + +CATEGORY_PATH_RE = re.compile(r"/cat/([^/?#]+)", re.IGNORECASE) + + +def parse_category_key(source_url: str) -> str: + parts = urlsplit(source_url.strip()) + query = parse_qs(parts.query) + if query.get("cat"): + return str(query["cat"][0]) + match = CATEGORY_PATH_RE.search(parts.path) + if match: + return match.group(1) + raise AppError(422, "SOURCE_URL_INVALID", "Quellpfad muss eine Geizhals-Kategorie enthalten (cat=…)") + + +def slugify(value: str) -> str: + text = value.strip().casefold().translate(str.maketrans("äöüß", "aous")) + text = re.sub(r"[^a-z0-9]+", "-", text).strip("-") + return text or "category" + + +def with_extended_filters(source_url: str) -> str: + parts = urlsplit(source_url.strip()) + query = [(name, value) for name, value in _query_items(parts.query) if name != "promode"] + query.append(("promode", "true")) + return urlunsplit((parts.scheme or "https", parts.netloc, parts.path or "/", urlencode(query), parts.fragment)) + + +def _query_items(query: str) -> list[tuple[str, str]]: + from urllib.parse import parse_qsl + + return parse_qsl(query, keep_blank_values=True) diff --git a/backend/app/sources/geizhals/live_page.py b/backend/app/sources/geizhals/live_page.py index 043ea50..a860ae9 100644 --- a/backend/app/sources/geizhals/live_page.py +++ b/backend/app/sources/geizhals/live_page.py @@ -15,6 +15,8 @@ BROWSER_HEADERS = { "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", } EXPAND_LABELS = ( + "Erweiterte Filter", + "Alle Filter ausklappen", "Immer alle Filter anzeigen", "weitere Filter anzeigen", ) @@ -42,19 +44,19 @@ def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, in def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]: status, html = 0, "" + if playwright: + try: + status, html = _fetch_playwright(url, expand_filters=True) + if not _blocked(status, html): + return html, "live_playwright", status + except Exception: + pass try: status, html = _fetch_http(url) if not _blocked(status, html): return html, "live_http", status except Exception: - status, html = 0, "" - if playwright: - try: - status, html = _fetch_playwright(url) - if not _blocked(status, html): - return html, "live_playwright", status - except Exception: - pass + status, html = status or 0, html or "" recorded = FIXTURE_DIR / f"{category_ref}-category.html" if recorded.exists(): return recorded.read_text(encoding="utf-8"), "recorded_html", 0 @@ -73,7 +75,7 @@ def _fetch_http(url: str) -> tuple[int, str]: return response.status_code, response.text -def _fetch_playwright(url: str) -> tuple[int, str]: +def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, str]: try: from playwright.sync_api import sync_playwright except ImportError as exc: @@ -87,17 +89,36 @@ def _fetch_playwright(url: str) -> tuple[int, str]: page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS) response = page.goto(url, wait_until="domcontentloaded", timeout=45000) page.wait_for_timeout(4000) + if expand_filters: + _expand_filter_sections(page) for label in EXPAND_LABELS: button = page.get_by_text(label, exact=False) if button.count(): - button.first.click(timeout=5000) - page.wait_for_timeout(1500) + try: + button.first.click(timeout=4000) + page.wait_for_timeout(800) + except Exception: + continue + if expand_filters: + _expand_filter_sections(page) html = page.content() status = response.status if response is not None else 200 browser.close() return status, html +def _expand_filter_sections(page: object) -> None: + locator = page.locator('button[aria-label*="aufklappen"]') + for _ in range(40): + if locator.count() == 0: + return + try: + locator.first.click(timeout=3000) + page.wait_for_timeout(700) + except Exception: + return + + def _blocked(status: int, html: str) -> bool: if status in {401, 403, 429}: return True diff --git a/backend/tests/unit/test_geizhals_category_url.py b/backend/tests/unit/test_geizhals_category_url.py new file mode 100644 index 0000000..8c0ea67 --- /dev/null +++ b/backend/tests/unit/test_geizhals_category_url.py @@ -0,0 +1,18 @@ +from app.errors import AppError +from app.sources.geizhals.category_url import parse_category_key, slugify, with_extended_filters +import pytest + + +def test_parse_category_key_from_query() -> None: + assert parse_category_key("https://geizhals.de/?cat=gra16_512&xf=1_AMD") == "gra16_512" + + +def test_parse_category_key_rejects_unknown_url() -> None: + with pytest.raises(AppError) as exc: + parse_category_key("https://geizhals.de/") + assert exc.value.code == "SOURCE_URL_INVALID" + + +def test_slugify_and_extended_filters() -> None: + assert slugify("Grafikkarten") == "grafikkarten" + assert "promode=true" in with_extended_filters("https://geizhals.de/?cat=nb") diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 8dab3a7..fd3b297 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -41,6 +41,8 @@ export const api = { sourceCategories: (source: string) => request<{ items: SourceCategory[] }>(`/sources/${source}/categories`), discoverTaxonomy: (source: string, category: string) => request(`/sources/${source}/categories/${category}/discover-taxonomy`, { method: "POST" }), + createSourceCategory: (source: string, payload: { source_url: string; name?: string }) => + request(`/sources/${source}/categories`, { method: "POST", body: JSON.stringify(payload) }), filters: (source: string, category: string) => request<{ filters: SourceFilter[] }>(`/sources/${source}/categories/${category}/filters`), }; @@ -166,5 +168,6 @@ export type SourceCategory = { status: string; taxonomy_status?: string; product_category_id?: string | null; + source_url?: string | null; }; export type SourceFilter = { label: string; external_filter_key: string; filter_type: string; status: string }; diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index 275a0e6..a21700d 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -1,4 +1,5 @@ import { useMutation, useQuery } from "@tanstack/react-query"; +import { useState } from "react"; import { api } from "../api"; import { EmptyState, LoadingState } from "../components/State"; @@ -10,36 +11,79 @@ export function AdminPage() { queryFn: () => api.sourceCategories(source), enabled: Boolean(source), }); - const category = categories.data?.items[0]?.external_category_key || ""; + const [category, setCategory] = useState(""); + const selectedKey = category || categories.data?.items[0]?.external_category_key || ""; const filters = useQuery({ - queryKey: ["filters", source, category], - queryFn: () => api.filters(source, category), - enabled: Boolean(source && category), + queryKey: ["filters", source, selectedKey], + queryFn: () => api.filters(source, selectedKey), + enabled: Boolean(source && selectedKey), }); + const [sourceUrl, setSourceUrl] = useState(""); + const [name, setName] = useState(""); const discover = useMutation({ - mutationFn: () => api.discoverTaxonomy(source, category), + mutationFn: () => api.discoverTaxonomy(source, selectedKey), onSuccess: () => { categories.refetch(); filters.refetch(); }, }); + const register = useMutation({ + mutationFn: () => api.createSourceCategory(source, { source_url: sourceUrl, name: name || undefined }), + onSuccess: (created) => { + setCategory(created.external_category_key); + setSourceUrl(""); + setName(""); + categories.refetch(); + }, + }); if (sources.isLoading) return ; - const selected = categories.data?.items.find((item) => item.external_category_key === category); + const selected = categories.data?.items.find((item) => item.external_category_key === selectedKey); return (

Quellen und Taxonomie

-

Technische IDs sind nur hier sichtbar.

+

+ Weitere Produktkategorien kommen über den Geizhals-Quellpfad. Beim Lernen werden zugeklappte Filtergruppen + geöffnet; das erste HTML enthält nur die Sektion „Allgemein“. +

+
+ + + {register.isError ?

{(register.error as Error).message}

: null} + +

Source: {source || "–"}

-

Kategorie: {category || "–"}

+ +

Quellpfad: {selected?.source_url || "–"}

Status: {selected?.taxonomy_status || "NOT_LEARNED"}

Gelernte Filter: {filters.data?.filters.length ?? 0}

{discover.isError ?

{(discover.error as Error).message}

: null} -
- {!category ? ( + {!selectedKey ? ( Keine Source-Kategorie hinterlegt. ) : !filters.data?.filters.length ? ( Taxonomie wurde noch nicht geladen.