From 46857715df768ff832aaf5491dd969b3d6d8fefa Mon Sep 17 00:00:00 2001 From: Lars Date: Sun, 4 Oct 2026 14:53:08 +0200 Subject: [PATCH] Lerne Taxonomie und Trefferliste live von Geizhals, damit Wizard und Kandidaten echte Quelldaten nutzen. Co-authored-by: Cursor --- backend/Dockerfile | 6 +- backend/app/discovery/urlgen.py | 81 ++- backend/app/services/catalog_service.py | 30 +- backend/app/services/discovery_service.py | 112 ++++- backend/app/settings.py | 2 + backend/app/sources/geizhals/adapter.py | 56 ++- backend/app/sources/geizhals/live_page.py | 104 ++++ backend/app/sources/geizhals/result_parser.py | 285 +++++++++++ .../app/sources/geizhals/taxonomy_parser.py | 474 ++++++++++++++++++ backend/fixtures/geizhals/nb-category.html | 132 +++++ backend/fixtures/geizhals/nb-search-page.html | 52 ++ backend/pyproject.toml | 4 + backend/tests/conftest.py | 13 + backend/tests/unit/test_discovery_urlgen.py | 27 + .../tests/unit/test_geizhals_result_parser.py | 42 ++ .../unit/test_geizhals_taxonomy_parser.py | 76 +++ backend/tests/unit/test_settings.py | 6 + backend/uv.lock | 96 +++- docker-compose.dev-env.yml | 3 + docker-compose.test.yml | 1 + docker-compose.yml | 2 + frontend/nginx.conf | 3 + frontend/src/api.ts | 9 +- frontend/src/pages/AdminPage.tsx | 3 +- frontend/src/pages/WizardPage.tsx | 22 +- 25 files changed, 1614 insertions(+), 27 deletions(-) create mode 100644 backend/app/sources/geizhals/live_page.py create mode 100644 backend/app/sources/geizhals/result_parser.py create mode 100644 backend/app/sources/geizhals/taxonomy_parser.py create mode 100644 backend/fixtures/geizhals/nb-category.html create mode 100644 backend/fixtures/geizhals/nb-search-page.html create mode 100644 backend/tests/unit/test_geizhals_result_parser.py create mode 100644 backend/tests/unit/test_geizhals_taxonomy_parser.py diff --git a/backend/Dockerfile b/backend/Dockerfile index a9b991e..7a8c461 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -18,11 +18,13 @@ RUN uv sync --frozen --no-dev --no-install-project FROM base AS runtime COPY . . -RUN uv sync --frozen --no-dev \ +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright +RUN uv sync --frozen --extra live --no-dev \ + && playwright install --with-deps chromium \ && sed -i 's/\r$//' /app/scripts/startup.sh \ && chmod +x /app/scripts/startup.sh \ && useradd --create-home --uid 1000 app \ - && chown -R app:app /app + && chown -R app:app /app /ms-playwright USER app EXPOSE 8000 diff --git a/backend/app/discovery/urlgen.py b/backend/app/discovery/urlgen.py index e7fa2d3..5b73888 100644 --- a/backend/app/discovery/urlgen.py +++ b/backend/app/discovery/urlgen.py @@ -27,11 +27,17 @@ def generate_search_url( base = urlsplit(category_url) query = dict(parse_qsl(base.query, keep_blank_values=True)) extra: list[tuple[str, str]] = [] + xf_values: list[str] = [] for name, value in fragments: - if name in query and name != "xf": + if name == "xf": + xf_values.append(value) + continue + if name in query: query[name] = value else: extra.append((name, value)) + if xf_values: + extra.append(("xf", "~".join(xf_values))) encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote) url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment)) return url, internal_only @@ -40,11 +46,15 @@ def generate_search_url( def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]: encoding = mapping.get("encoding") or {} param = str(encoding.get("url_param") or "xf") + if encoding.get("encoding_type") == "RANGE_PARAMETERS" or param in {"bpmin", "bpmax"}: + return _encode_range(encoding, criterion) + if encoding.get("encoding_type") == "RANGE_TOKEN": + return _encode_range_token(mapping, criterion) if param == "xf": filter_key = str(mapping["external_filter_key"]) - source_value = str(mapping.get("source_value") or _threshold_value(mapping, criterion)) + source_value = _xf_value(mapping, criterion) return [("xf", f"{filter_key}_{_plus(source_value)}")] - if param in {"minp", "maxp", "prmin", "prmax"}: + if param in {"minp", "maxp", "prmin", "prmax", "asuch", "fs"}: return [(param, str(criterion.get("value")))] template = encoding.get("value_template") if template: @@ -52,6 +62,63 @@ def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[ raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}") +def _encode_range(encoding: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]: + operator = str(criterion.get("operator") or "<=") + value = criterion.get("value") + min_param = str(encoding.get("min_parameter") or encoding.get("url_param") or "bpmin") + max_param = str(encoding.get("max_parameter") or "bpmax") + if operator == "between": + if isinstance(value, dict): + items: list[tuple[str, str]] = [] + if value.get("min") is not None: + items.append((min_param, str(value["min"]))) + if value.get("max") is not None: + items.append((max_param, str(value["max"]))) + return items + if isinstance(value, list) and len(value) == 2: + return [(min_param, str(value[0])), (max_param, str(value[1]))] + if operator in {">=", ">"}: + return [(min_param, str(value))] + return [(max_param, str(value))] + + +def _encode_range_token(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]: + filter_key = str(mapping["external_filter_key"]) + looked_up = _lookup_option(mapping, criterion.get("value")) + if looked_up is not None: + return [("xf", f"{filter_key}_{_plus(looked_up)}")] + operator = str(criterion.get("operator") or "<=") + value = criterion.get("value") + if operator == "between": + if isinstance(value, dict): + raw = f"{value.get('min') or ''}*{value.get('max') or ''}" + elif isinstance(value, list) and len(value) == 2: + raw = f"{value[0]}*{value[1]}" + else: + raw = f"*{value}" + elif operator in {">=", ">"}: + raw = f"{value}*" + else: + raw = f"*{value}" + return [("xf", f"{filter_key}_{raw}")] + + +def _lookup_option(mapping: dict[str, Any], value: Any) -> str | None: + options = (mapping.get("encoding") or {}).get("options") or {} + if value in options: + return str(options[value]) + if str(value) in options: + return str(options[str(value)]) + return None + + +def _xf_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str: + looked_up = _lookup_option(mapping, criterion.get("value")) + if looked_up is not None: + return looked_up + return str(mapping.get("source_value") or _threshold_value(mapping, criterion)) + + def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str: option = mapping.get("source_value") if option: @@ -72,8 +139,12 @@ def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]: parts = urlsplit(url) decoded: list[tuple[str, str]] = [] for name, value in parse_qsl(parts.query, keep_blank_values=True): - if name == "xf" and "_" in value: - key, raw = value.split("_", 1) + if name != "xf": + continue + for part in value.replace("|", "~").split("~"): + if "_" not in part: + continue + key, raw = part.split("_", 1) decoded.append((key, raw.replace("+", " "))) return decoded diff --git a/backend/app/services/catalog_service.py b/backend/app/services/catalog_service.py index 3029d60..27d519c 100644 --- a/backend/app/services/catalog_service.py +++ b/backend/app/services/catalog_service.py @@ -100,7 +100,7 @@ class CatalogService: SourceCategoryRecord.product_category_id == product_category_id, ) ).first() - if source_category is not None and source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value: + if source_category is not None and self._should_learn(source_category, source_id): from app.services.discovery_service import TaxonomyService TaxonomyService(self.session).discover(source_id, source_category.external_category_key) @@ -172,7 +172,33 @@ class CatalogService: if definition.id in seen: continue criteria.append(self._criterion_out(definition, None, None, [], None)) - return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria}) + return jsonable( + { + "product_category_id": product_category_id, + "source": source_id, + "taxonomy_status": None if source_category is None else source_category.taxonomy_status, + "criteria": criteria, + } + ) + + def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool: + if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value: + return True + if source_category.taxonomy_status != TaxonomyStatus.READY.value: + return False + from app.settings import get_settings + + if get_settings().geizhals_use_fixtures or source_id != "geizhals": + return False + keys = { + row.external_filter_key + for row in self.session.scalars( + select(SourceFilterDefinitionRecord).where( + SourceFilterDefinitionRecord.source_category_id == source_category.id + ) + ) + } + return keys <= {"17023", "price"} def list_templates(self) -> list[dict[str, Any]]: rows, _total = self.catalog.list_profiles(200, 0) diff --git a/backend/app/services/discovery_service.py b/backend/app/services/discovery_service.py index 2bc0137..e81dea3 100644 --- a/backend/app/services/discovery_service.py +++ b/backend/app/services/discovery_service.py @@ -14,6 +14,7 @@ from app.discovery.diff import diff_run, next_lifecycle from app.discovery.enums import ( CandidateStatus, DiffStatus, + FilterType, LifecycleStatus, MappingStatus, MatchStatus, @@ -99,7 +100,20 @@ class TaxonomyService: if catalog.get_source(source_id) is None: raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) adapter = self.adapter or discovery_adapter(source_id) - payload = adapter.discover_taxonomy(category_key) + try: + payload = adapter.discover_taxonomy(category_key) + except AppError as exc: + category = self.session.scalars( + select(SourceCategoryRecord).where( + SourceCategoryRecord.source_id == source_id, + SourceCategoryRecord.external_category_key == category_key, + ) + ).first() + if category is not None and exc.code == "SOURCE_BLOCKED": + category.taxonomy_status = TaxonomyStatus.BLOCKED.value + category.last_taxonomy_scan_at = now_utc() + self.session.flush() + raise now = now_utc() category = self.session.scalars( select(SourceCategoryRecord).where( @@ -148,7 +162,7 @@ class TaxonomyService: label=str(raw.get("label") or raw["external_filter_key"]), filter_type=filter_type, canonical_unit=raw.get("canonical_unit"), - status=MappingStatus.VERIFIED.value, + status=MappingStatus.LEARNED_UNVERIFIED.value, ) self.session.add(filt) self.session.flush() @@ -161,9 +175,14 @@ class TaxonomyService: self._sync_options(filt, raw.get("options") or []) if raw.get("range"): self._sync_range(filt, raw["range"]) - category.taxonomy_status = ( - TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value - ) + self._sync_learned_mapping(category, filt, raw) + requested = str(payload.get("taxonomy_status") or "") + if requested in {item.value for item in TaxonomyStatus}: + category.taxonomy_status = requested + else: + category.taxonomy_status = ( + TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value + ) self.session.flush() return self.filters(source_id, category.external_category_key) @@ -275,6 +294,89 @@ class TaxonomyService: if removed: self._break_mappings(filt.source_category_id, filt.external_filter_key) + def _sync_learned_mapping( + self, + category: SourceCategoryRecord, + filt: SourceFilterDefinitionRecord, + raw: dict[str, Any], + ) -> None: + encoding = raw.get("encoding") or {"url_param": "xf"} + definition = upsert_definition( + self.session, + { + "key": f"{category.source_id}:{category.external_category_key}:{filt.external_filter_key}", + "origin": "SOURCE", + "source": category.source_id, + "product_category_id": category.product_category_id, + "label": filt.label, + "value_type": "number" + if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value} + else "string", + "unit": filt.canonical_unit, + "category": category.product_category_id, + }, + ) + mapping = self.session.scalars( + select(CriterionMappingRecord).where( + CriterionMappingRecord.source_category_id == category.id, + CriterionMappingRecord.external_filter_key == filt.external_filter_key, + ) + ).first() + options = raw.get("options") or [] + source_value = None + if options: + source_value = str(options[0].get("query_value") or options[0].get("source_value")) + now = now_utc() + if mapping is None: + mapping = CriterionMappingRecord( + id=uuid4(), + criterion_definition_id=definition.id, + criterion_key=definition.key, + source_id=category.source_id, + source_category_id=category.id, + external_filter_key=filt.external_filter_key, + source_value=source_value, + encoding=encoding, + status=MappingStatus.LEARNED_UNVERIFIED.value, + definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})), + created_at=now, + updated_at=now, + ) + self.session.add(mapping) + self.session.flush() + else: + mapping.criterion_definition_id = definition.id + mapping.criterion_key = definition.key + mapping.source_value = source_value + mapping.encoding = encoding + mapping.definition_hash = sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})) + mapping.updated_at = now + probe_value = source_value or 0 + if options and options[0].get("normalized_value") is not None: + probe_value = options[0]["normalized_value"] + operator = (options[0].get("operator") if options else None) or "<=" + if filt.filter_type == FilterType.NUMERIC_RANGE.value: + operator = ">=" + probe_value = 1000 + verified = verify_roundtrip( + category_url=category.source_url or f"https://geizhals.de/?cat={category.external_category_key}", + criterion={ + "criterion": definition.key, + "operator": operator, + "value": probe_value, + }, + mapping={ + "criterion_key": definition.key, + "external_filter_key": filt.external_filter_key, + "source_value": source_value, + "encoding": encoding, + }, + ) + if mapping.status != MappingStatus.DISABLED.value and verified: + mapping.status = MappingStatus.VERIFIED.value + elif mapping.status == MappingStatus.BROKEN.value: + mapping.status = MappingStatus.LEARNED_UNVERIFIED.value + def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None: row = self.session.get(SourceFilterRangeRecord, filt.id) if row is None: diff --git a/backend/app/settings.py b/backend/app/settings.py index bd04c9b..d0947fa 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -16,6 +16,8 @@ class Settings(BaseSettings): engine_build: str = "dev" clock_now: datetime | None = None cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173" + geizhals_use_fixtures: bool = False + geizhals_playwright: bool = True @property def cors_origins(self) -> list[str]: diff --git a/backend/app/sources/geizhals/adapter.py b/backend/app/sources/geizhals/adapter.py index dcbc3d0..b140afd 100644 --- a/backend/app/sources/geizhals/adapter.py +++ b/backend/app/sources/geizhals/adapter.py @@ -7,6 +7,10 @@ from urllib.parse import parse_qs, urlparse from app.discovery.enums import FilterType from app.errors import AppError +from app.settings import get_settings +from app.sources.geizhals.live_page import load_category_page, load_result_page +from app.sources.geizhals.result_parser import parse_result_html, with_page +from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals" @@ -14,12 +18,31 @@ FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals" class GeizhalsAdapter: source_key = "geizhals" - def __init__(self, fixture_dir: Path | None = None, live: bool = False) -> None: + def __init__(self, fixture_dir: Path | None = None, live: bool | None = None) -> None: self.fixture_dir = fixture_dir or FIXTURE_DIR - self.live = live + settings = get_settings() + self.live = (not settings.geizhals_use_fixtures) if live is None else live def discover_taxonomy(self, category_ref: str) -> dict[str, Any]: - payload = self._load(f"{category_ref}-taxonomy.json") + if not self.live: + return self._load(f"{category_ref}-taxonomy.json") + url = f"https://geizhals.de/?cat={category_ref}&promode=true" + html, mode, status = load_category_page( + url, + category_ref=category_ref, + playwright=get_settings().geizhals_playwright, + ) + payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=url) + if payload.get("blocked") or not payload.get("filters"): + raise AppError( + 422, + "TAXONOMY_EMPTY", + "Geizhals-Taxonomie konnte nicht aus der Kategorieseite gelesen werden", + {"url": url, "http_status": status, "mode": mode}, + ) + payload["discovery_mode"] = mode + payload["http_status"] = status + payload["taxonomy_status"] = "DEGRADED" if mode == "recorded_html" else "READY" return payload def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: @@ -34,13 +57,32 @@ class GeizhalsAdapter: return {"url": url, "internal_only": internal_only} def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]: - if self.live: - raise AppError(422, "LIVE_FETCH_DISABLED", "live Geizhals fetch is not enabled in this process") page = int(pagination.get("page") or 1) - category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0] - return self._load(f"{category}-search-page{page}.json") + if not self.live: + category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0] + return self._load(f"{category}-search-page{page}.json") + url = with_page(str(query["url"]), page) + html, mode, status, blocked = load_result_page(url, playwright=get_settings().geizhals_playwright) + if blocked: + return { + "page": page, + "page_count": page, + "result_count": 0, + "http_status": status or 403, + "blocked": True, + "challenge": "captcha" if status in {401, 403} else None, + "items": [], + "discovery_mode": mode, + } + payload = parse_result_html(html, page_url=url) + payload["page"] = page + payload["http_status"] = status + payload["discovery_mode"] = mode + return payload def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]: + if response.get("html") and not response.get("items"): + return list(parse_result_html(str(response["html"]), page_url=str(response.get("url") or "")).get("items") or []) return list(response.get("items") or []) def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/app/sources/geizhals/live_page.py b/backend/app/sources/geizhals/live_page.py new file mode 100644 index 0000000..043ea50 --- /dev/null +++ b/backend/app/sources/geizhals/live_page.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from pathlib import Path + +from app.errors import AppError +from app.sources.geizhals.taxonomy_parser import is_challenge_html + +FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals" +BROWSER_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", +} +EXPAND_LABELS = ( + "Immer alle Filter anzeigen", + "weitere Filter anzeigen", +) + + +def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, int, bool]: + status, html = 0, "" + try: + status, html = _fetch_http(url) + if not _blocked(status, html): + return html, "live_http", status, False + except Exception: + status, html = 0, "" + if playwright: + try: + status, html = _fetch_playwright(url) + if not _blocked(status, html): + return html, "live_playwright", status, False + except Exception: + pass + if html and not is_challenge_html(html) and status and status < 400: + return html, "live_http", status, False + return html, "blocked", status or 403, True + + +def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]: + status, html = 0, "" + try: + status, html = _fetch_http(url) + if not _blocked(status, html): + return html, "live_http", status + except Exception: + status, html = 0, "" + if playwright: + try: + status, html = _fetch_playwright(url) + if not _blocked(status, html): + return html, "live_playwright", status + except Exception: + pass + recorded = FIXTURE_DIR / f"{category_ref}-category.html" + if recorded.exists(): + return recorded.read_text(encoding="utf-8"), "recorded_html", 0 + raise AppError( + 423, + "SOURCE_BLOCKED", + "Geizhals blockiert den Taxonomie-Abruf (403/Challenge)", + {"url": url, "http_status": status}, + ) + + +def _fetch_http(url: str) -> tuple[int, str]: + import httpx + + response = httpx.get(url, headers=BROWSER_HEADERS, follow_redirects=True, timeout=25.0) + return response.status_code, response.text + + +def _fetch_playwright(url: str) -> tuple[int, str]: + try: + from playwright.sync_api import sync_playwright + except ImportError as exc: + raise AppError( + 423, + "SOURCE_BLOCKED", + "Geizhals verlangt einen Browser-Abruf, Playwright ist nicht installiert", + ) from exc + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS) + response = page.goto(url, wait_until="domcontentloaded", timeout=45000) + page.wait_for_timeout(4000) + for label in EXPAND_LABELS: + button = page.get_by_text(label, exact=False) + if button.count(): + button.first.click(timeout=5000) + page.wait_for_timeout(1500) + html = page.content() + status = response.status if response is not None else 200 + browser.close() + return status, html + + +def _blocked(status: int, html: str) -> bool: + if status in {401, 403, 429}: + return True + return is_challenge_html(html) diff --git a/backend/app/sources/geizhals/result_parser.py b/backend/app/sources/geizhals/result_parser.py new file mode 100644 index 0000000..4caefd5 --- /dev/null +++ b/backend/app/sources/geizhals/result_parser.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import re +from typing import Any +from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit + +from app.sources.geizhals.taxonomy_parser import is_challenge_html, node_text, parse_html_tree + +PRODUCT_ID_RE = re.compile(r"-a(\d+)\.html", re.IGNORECASE) +PRICE_RE = re.compile(r"(\d{1,3}(?:\.\d{3})+|\d+),(\d{2})") +OFFER_RE = re.compile(r"(\d+)\s*Angebote?", re.IGNORECASE) +PRODUCT_COUNT_RE = re.compile(r"(\d[\d.]*)\s*Produkte", re.IGNORECASE) +CAPACITY_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*(TB|GB)", re.IGNORECASE) +WEIGHT_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*kg", re.IGNORECASE) +AVAILABILITY_BY_CODE = {"0": "out_of_stock", "1": "limited", "2": "in_stock"} +AVAILABILITY_BY_LABEL = { + "lagernd": "in_stock", + "kurzfristig lieferbar": "limited", + "kurzfristig": "limited", + "nicht verfügbar": "out_of_stock", +} + + +def parse_result_html(html: str, *, page_url: str) -> dict[str, Any]: + if is_challenge_html(html): + return {"items": [], "page_count": 1, "result_count": 0, "blocked": True} + root = parse_html_tree(html) + cards = _find_cards(root) + items: list[dict[str, Any]] = [] + seen: set[str] = set() + for card in cards: + item = _parse_card(card, page_url) + if item is None or item["source_external_id"] in seen: + continue + seen.add(item["source_external_id"]) + items.append(item) + page_count = _page_count(root) + result_count = _result_count(root) or len(items) + return { + "items": items, + "page_count": page_count, + "result_count": result_count, + "blocked": False, + } + + +def with_page(url: str, page: int) -> str: + parts = urlsplit(url) + query = [(name, value) for name, value in parse_qsl(parts.query, keep_blank_values=True) if name != "pg"] + if page > 1: + query.append(("pg", str(page))) + return parts._replace(query=urlencode(query, doseq=True)).geturl() + + +def _find_cards(root: Any) -> list[Any]: + cards: list[Any] = [] + + def walk(node: Any) -> None: + classes = node.attrs.get("class", "") + if "galleryview__item" in classes or "listview__item" in classes: + cards.append(node) + return + for child in node.children: + if not isinstance(child, str): + walk(child) + + walk(root) + return cards + + +def _parse_card(card: Any, page_url: str) -> dict[str, Any] | None: + link = _first_link(card, "name-link") or _first_product_link(card) + if link is None: + return None + href = link.attrs.get("href") or "" + match = PRODUCT_ID_RE.search(href) + if match is None: + return None + title = (link.attrs.get("title") or node_text(link)).strip() + if not title: + return None + source_url = urljoin(page_url, href.split("#")[0]) + raw_attributes = _definition_list(card) + sku = _first_text_by_class(card, "galleryview__mpn") or _first_text_by_class(card, "listview__mpn") + price_text = _first_text_by_class(card, "price") + offers_text = _first_text_by_class(card, "offercount") or "" + availability_node = _first_by_attr(card, "data-availability") + manufacturer, model = _split_title(title) + return { + "source_external_id": match.group(1), + "source_url": source_url, + "source_title": re.sub(r"\s+", " ", title), + "manufacturer": manufacturer, + "model": model, + "manufacturer_sku": sku, + "gtin_ean": None, + "best_price": _parse_price(price_text), + "currency": "EUR" if price_text and "€" in price_text else None, + "availability": _availability(availability_node), + "seller_count": _seller_count(offers_text), + "raw_attributes": raw_attributes, + "normalized_attributes": _normalize_attributes(raw_attributes), + } + + +def _first_link(node: Any, fragment: str) -> Any | None: + classes = node.attrs.get("class", "") + if node.tag == "a" and fragment in classes: + return node + for child in node.children: + if not isinstance(child, str): + found = _first_link(child, fragment) + if found is not None: + return found + return None + + +def _first_product_link(node: Any) -> Any | None: + if node.tag == "a" and PRODUCT_ID_RE.search(node.attrs.get("href") or ""): + href = node.attrs.get("href") or "" + if "#ratings" not in href and "#offerlist" not in href: + return node + for child in node.children: + if not isinstance(child, str): + found = _first_product_link(child) + if found is not None: + return found + return None + + +def _first_text_by_class(node: Any, fragment: str) -> str | None: + found = _first_by_class(node, fragment) + if found is None: + return None + text = node_text(found).strip() + return re.sub(r"\s+", " ", text) or None + + +def _first_by_class(node: Any, fragment: str) -> Any | None: + if fragment in node.attrs.get("class", ""): + return node + for child in node.children: + if not isinstance(child, str): + found = _first_by_class(child, fragment) + if found is not None: + return found + return None + + +def _first_by_attr(node: Any, name: str) -> Any | None: + if name in node.attrs: + return node + for child in node.children: + if not isinstance(child, str): + found = _first_by_attr(child, name) + if found is not None: + return found + return None + + +def _definition_list(node: Any) -> dict[str, str]: + pairs: dict[str, str] = {} + + def walk(current: Any) -> None: + if current.tag == "dl": + pending: str | None = None + for child in current.children: + if isinstance(child, str): + continue + if child.tag == "dt": + pending = re.sub(r"\s+", " ", node_text(child)).strip() + elif child.tag == "dd" and pending: + pairs[pending] = re.sub(r"\s+", " ", node_text(child)).strip() + pending = None + return + for child in current.children: + if not isinstance(child, str): + walk(child) + + walk(node) + return pairs + + +def _split_title(title: str) -> tuple[str | None, str | None]: + parts = title.split(None, 1) + if not parts: + return None, None + if len(parts) == 1: + return parts[0], None + return parts[0], parts[1] + + +def _parse_price(text: str | None) -> str | None: + if not text: + return None + match = PRICE_RE.search(text.replace("\xa0", " ")) + if match is None: + return None + whole = match.group(1).replace(".", "") + return f"{whole}.{match.group(2)}" + + +def _seller_count(text: str) -> int | None: + match = OFFER_RE.search(text) + if match is None: + return None + return int(match.group(1)) + + +def _availability(node: Any | None) -> str: + if node is None: + return "unknown" + code = AVAILABILITY_BY_CODE.get(node.attrs.get("data-availability", "")) + if code: + return code + label = (node.attrs.get("aria-label") or node_text(node)).strip().casefold() + return AVAILABILITY_BY_LABEL.get(label, "unknown") + + +def _normalize_attributes(raw: dict[str, str]) -> dict[str, Any]: + normalized: dict[str, Any] = {} + for label, value in raw.items(): + key = label.casefold() + capacity = _capacity_gb(value) + if capacity is not None and key in {"ram", "arbeitsspeicher"}: + normalized["ram_gb"] = capacity + elif capacity is not None and key in {"speicher", "ssd", "festplatte"}: + normalized["storage_gb"] = capacity + weight = _weight_kg(value) + if weight is not None and "gewicht" in key: + normalized["weight_kg"] = weight + return normalized + + +def _capacity_gb(text: str) -> float | int | None: + match = CAPACITY_RE.search(text) + if match is None: + return None + value = float(match.group(1).replace(",", ".")) + if match.group(2).upper() == "TB": + value *= 1024 + return int(value) if value.is_integer() else value + + +def _weight_kg(text: str) -> float | None: + match = WEIGHT_RE.search(text) + if match is None: + return None + return float(match.group(1).replace(",", ".")) + + +def _page_count(root: Any) -> int: + pages = {1} + + def walk(node: Any) -> None: + href = node.attrs.get("href") or "" + if "pg=" in href and ("pagination" in node.attrs.get("class", "") or _in_pagination(node)): + for name, value in parse_qsl(urlsplit(href).query): + if name == "pg" and value.isdigit(): + pages.add(int(value)) + for child in node.children: + if not isinstance(child, str): + walk(child) + + walk(root) + return max(pages) + + +def _in_pagination(node: Any) -> bool: + current = node + while current is not None: + classes = current.attrs.get("class", "") + ident = current.attrs.get("id", "") + if "pagination" in classes or ident == "pagination": + return True + current = current.parent + return False + + +def _result_count(root: Any) -> int | None: + counts = [ + int(match.replace(".", "")) + for match in PRODUCT_COUNT_RE.findall(node_text(root)) + ] + return max(counts) if counts else None diff --git a/backend/app/sources/geizhals/taxonomy_parser.py b/backend/app/sources/geizhals/taxonomy_parser.py new file mode 100644 index 0000000..6c91659 --- /dev/null +++ b/backend/app/sources/geizhals/taxonomy_parser.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import re +from html.parser import HTMLParser +from typing import Any +from urllib.parse import parse_qsl, unquote, urlparse + +from app.discovery.enums import FilterType + +SKIP_OPTION_LABELS = { + "alle", + "entfernen", + "überspringen", + "aktualisieren", + "abonnieren", + "filter zurücksetzen", + "nur beliebteste filter anzeigen", + "immer alle filter anzeigen", + "filter überspringen", + "einfache filter", + "erweiterte filter", +} +RANGE_MIN_NAMES = {"bpmin", "minp", "prmin"} +RANGE_MAX_NAMES = {"bpmax", "maxp", "prmax"} +TEXT_NAMES = {"asuch", "fs"} +HEADING_TAGS = {"th", "h2", "h3", "h4", "h5", "legend", "dt", "caption", "label", "strong"} +NUMBER_RE = re.compile(r"(-?\d+(?:[.,]\d+)?)") +THRESHOLD_RE = re.compile(r"^(ab|bis)\s+", re.IGNORECASE) +UNIT_RE = re.compile(r"(GB|TB|GHz|MHz|Hz|cd/m²|ppi|kg|\"|″|mm|%|€|EUR)$", re.IGNORECASE) +TYPE_MAP = { + "text": FilterType.ENUM.value, + "color": FilterType.ENUM.value, + "text_range": FilterType.NUMERIC_THRESHOLD.value, + "numeric_range": FilterType.NUMERIC_RANGE.value, +} + + +class _Node: + def __init__(self, tag: str, attrs: dict[str, str]) -> None: + self.tag = tag + self.attrs = attrs + self.children: list[_Node | str] = [] + self.parent: _Node | None = None + + +class _TreeBuilder(HTMLParser): + VOID = { + "area", + "base", + "br", + "col", + "embed", + "hr", + "img", + "input", + "link", + "meta", + "param", + "source", + "track", + "wbr", + } + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.root = _Node("document", {}) + self._stack = [self.root] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + node = _Node(tag, {key: value or "" for key, value in attrs}) + parent = self._stack[-1] + node.parent = parent + parent.children.append(node) + if tag not in self.VOID: + self._stack.append(node) + + def handle_endtag(self, tag: str) -> None: + for index in range(len(self._stack) - 1, 0, -1): + if self._stack[index].tag == tag: + del self._stack[index:] + break + + def handle_data(self, data: str) -> None: + text = data.strip() + if text: + self._stack[-1].children.append(text) + + +def parse_html_tree(html: str) -> _Node: + builder = _TreeBuilder() + builder.feed(html) + return builder.root + + +def node_text(node: _Node) -> str: + return _node_text(node) + + +def parse_taxonomy_html(html: str, *, category_ref: str, source_url: str) -> dict[str, Any]: + if is_challenge_html(html): + return { + "source": "geizhals", + "external_category_key": category_ref, + "name": category_ref, + "source_url": source_url, + "filters": [], + "blocked": True, + } + builder = _TreeBuilder() + builder.feed(html) + groups: dict[str, dict[str, Any]] = {} + _collect_declared_filters(builder.root, groups) + _collect_xf_links(builder.root, groups) + _collect_named_inputs(builder.root, groups) + filters = [ + group + for group in groups.values() + if _usable_group(group) + ] + filters.sort(key=lambda item: (item["label"].lower(), item["external_filter_key"])) + return { + "source": "geizhals", + "external_category_key": category_ref, + "name": _category_name(builder.root, category_ref), + "canonical_category": "notebook" if category_ref == "nb" else None, + "source_url": source_url, + "filters": filters, + "blocked": False, + } + + +def is_challenge_html(html: str) -> bool: + sample = html.lower() + markers = ( + "cf-mitigated", + "cf-challenge", + "just a moment", + "enable javascript and cookies", + "checking your browser", + "verification successful. waiting for geizhals", + ) + return any(marker in sample for marker in markers) + + +def _collect_declared_filters(node: _Node, groups: dict[str, dict[str, Any]]) -> None: + filter_id = node.attrs.get("data-filter-id") + if filter_id: + source_type = node.attrs.get("data-filter-type") or "" + label = _declared_label(node) or f"Filter {filter_id}" + group = groups.setdefault(filter_id, _empty_group(filter_id, label, source_type)) + if label and group["label"].startswith("Filter "): + group["label"] = label + if source_type: + group["filter_type"] = TYPE_MAP.get(source_type, group["filter_type"]) + if source_type == "numeric_range": + group["encoding"]["encoding_type"] = "RANGE_TOKEN" + group["encoding"]["value_template"] = "{min}*{max}" + group.setdefault("range", {"minimum": None, "maximum": None, "step": None, "unit": group.get("canonical_unit")}) + _collect_xf_links(node, groups, required_key=filter_id, allow_histogram=source_type == "numeric_range") + return + for child in node.children: + if isinstance(child, _Node): + _collect_declared_filters(child, groups) + + +def _collect_xf_links( + node: _Node, + groups: dict[str, dict[str, Any]], + required_key: str | None = None, + allow_histogram: bool = False, +) -> None: + if node.tag == "a": + href = node.attrs.get("href") or "" + label = node.attrs.get("aria-label") or _node_text(node) + if _usable_option(node, label, allow_histogram=allow_histogram): + for key, query_value in _xf_tokens(href): + if required_key is not None and key != required_key: + continue + if required_key is None and not _has_structural_label(node): + continue + group_label = _declared_label(_filter_root(node)) or _nearest_group_label(node) + group = groups.setdefault(key, _empty_group(key, group_label, None)) + if group_label and (group["label"].startswith("Filter ") or group["label"] == key): + group["label"] = group_label + option = _option_from_link(label, query_value) + existing = {item["source_value"] for item in group["options"]} + if option["source_value"] not in existing: + group["options"].append(option) + group["encoding"].setdefault("options", {}) + group["encoding"]["options"][option["source_value"]] = query_value + if option.get("normalized_value") is not None: + group["encoding"]["options"][str(option["normalized_value"])] = query_value + if group["filter_type"] not in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}: + group["filter_type"] = _classify_options(group["options"]) + if group["canonical_unit"] is None: + group["canonical_unit"] = option.get("unit") + for child in node.children: + if isinstance(child, _Node): + _collect_xf_links(child, groups, required_key=required_key, allow_histogram=allow_histogram) + + +def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> None: + inputs: list[_Node] = [] + _find_inputs(node, inputs) + names = {item.attrs.get("name", ""): item for item in inputs if item.attrs.get("name")} + if names.keys() & RANGE_MIN_NAMES or names.keys() & RANGE_MAX_NAMES: + min_name = next((name for name in RANGE_MIN_NAMES if name in names), "bpmin") + max_name = next((name for name in RANGE_MAX_NAMES if name in names), "bpmax") + label = _nearest_group_label(names.get(min_name) or names.get(max_name) or node) or "Preis" + groups["price"] = { + "external_filter_key": "price", + "label": "Preis" if "preis" not in label.lower() else label, + "filter_type": FilterType.NUMERIC_RANGE.value, + "canonical_unit": "EUR", + "options": [], + "range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"}, + "encoding": { + "url_param": min_name, + "min_parameter": min_name, + "max_parameter": max_name, + "encoding_type": "RANGE_PARAMETERS", + }, + } + for name in TEXT_NAMES: + if name not in names: + continue + label = _nearest_group_label(names[name]) or "Suchbegriffe" + groups[name] = { + "external_filter_key": name, + "label": label, + "filter_type": FilterType.TEXT.value, + "canonical_unit": None, + "options": [], + "encoding": {"url_param": name, "encoding_type": "TEXT_PARAMETER", "value_template": "{value}"}, + } + + +def _find_inputs(node: _Node, found: list[_Node]) -> None: + if node.tag == "input": + found.append(node) + for child in node.children: + if isinstance(child, _Node): + _find_inputs(child, found) + + +def _xf_tokens(href: str) -> list[tuple[str, str]]: + if not href or href.startswith("#") or href.startswith("javascript:"): + return [] + query = urlparse(href).query + tokens: list[tuple[str, str]] = [] + for name, value in parse_qsl(query, keep_blank_values=True): + if name != "xf" or not value: + continue + for part in unquote(value).replace("|", "~").split("~"): + if "_" not in part: + continue + key, raw = part.split("_", 1) + if key.isdigit() and raw: + tokens.append((key, raw.replace("+", " "))) + return tokens + + +def _option_from_link(label: str, query_value: str) -> dict[str, Any]: + cleaned = re.sub(r"\(\d[\d.]*\s*(produkte)?\)\s*$", "", label, flags=re.IGNORECASE).strip() + cleaned = re.sub(r"\s+", " ", cleaned) + operator = None + if cleaned.lower().startswith("ab "): + operator = ">=" + elif cleaned.lower().startswith("bis "): + operator = "<=" + number = _first_number(cleaned) + unit = _unit(cleaned) + return { + "source_value": cleaned or query_value, + "query_value": query_value, + "normalized_value": number, + "operator": operator, + "unit": unit, + } + + +def _classify_options(options: list[dict[str, Any]]) -> str: + if not options: + return FilterType.UNKNOWN.value + labels = [str(item["source_value"]).lower() for item in options] + if all(item in {"ja", "nein", "egal", "true", "false"} for item in labels): + return FilterType.BOOLEAN.value + if all(THRESHOLD_RE.match(str(item["source_value"])) for item in options): + return FilterType.NUMERIC_THRESHOLD.value + return FilterType.ENUM.value + + +def _empty_group(key: str, label: str | None, source_type: str | None) -> dict[str, Any]: + encoding_type = "OPTION_TOKEN" + if source_type == "numeric_range": + encoding_type = "RANGE_TOKEN" + return { + "external_filter_key": key, + "label": label or f"Filter {key}", + "filter_type": TYPE_MAP.get(source_type or "", FilterType.ENUM.value), + "canonical_unit": None, + "options": [], + "encoding": { + "url_param": "xf", + "encoding_type": encoding_type, + "options": {}, + **({"value_template": "{min}*{max}"} if encoding_type == "RANGE_TOKEN" else {}), + }, + } + + +def _declared_label(node: _Node | None) -> str | None: + if node is None: + return None + attr_id = node.attrs.get("id", "") + if attr_id.startswith("filter-element__title-label"): + return _clean_label(_node_text(node)) or None + for child in node.children: + if isinstance(child, _Node): + found = _declared_label(child) + if found: + return found + return None + + +def _filter_root(node: _Node) -> _Node | None: + current: _Node | None = node + while current is not None: + if current.attrs.get("data-filter-id"): + return current + current = current.parent + return None + + +def _nearest_group_label(node: _Node | None) -> str | None: + current = node + while current is not None: + declared = _declared_label(current) + if declared: + return declared + if current.tag in HEADING_TAGS: + text = _clean_label(_node_text(current)) + if text: + return text + previous = _previous_element(current) + while previous is not None: + if previous.tag in HEADING_TAGS: + text = _clean_label(_node_text(previous)) + if text: + return text + previous = _previous_element(previous) + labeled = current.attrs.get("aria-label") or current.attrs.get("data-filter-name") or current.attrs.get("title") + if labeled: + text = _clean_label(labeled) + if text and len(text.split()) <= 6: + return text + current = current.parent + return None + + +def _has_structural_label(node: _Node) -> bool: + if _filter_root(node) is not None: + return True + current: _Node | None = node + while current is not None: + if current.tag in {"th", "td", "tr", "table", "dt", "dd"}: + return _nearest_group_label(node) is not None + current = current.parent + return False + + +def _usable_option(node: _Node, label: str, *, allow_histogram: bool) -> bool: + classes = f"{node.attrs.get('class', '')} {node.attrs.get('id', '')}".lower() + if "skip-link" in classes or "add-to-columns" in classes: + return False + if "bucket" in classes and not allow_histogram: + return False + cleaned = re.sub(r"\s+", " ", label).strip() + if not cleaned or _skip_option(cleaned): + return False + if "überspringen" in cleaned.lower(): + return False + if not allow_histogram and "produkte" in cleaned.lower(): + return False + if len(cleaned.split()) > 8: + return False + return True + + +def _usable_group(group: dict[str, Any]) -> bool: + if group["filter_type"] == FilterType.TEXT.value: + return True + if group.get("range") and group["filter_type"] == FilterType.NUMERIC_RANGE.value: + return True + if not group.get("options") and not group.get("range"): + return False + label = str(group.get("label") or "") + if label.startswith("Filter ") and len(group.get("options") or []) < 2: + return False + if len(label.split()) > 8: + return False + return True + + +def _previous_element(node: _Node) -> _Node | None: + if node.parent is None: + return None + siblings = [child for child in node.parent.children if isinstance(child, _Node)] + try: + index = siblings.index(node) + except ValueError: + return None + return siblings[index - 1] if index > 0 else None + + +def _node_text(node: _Node) -> str: + parts: list[str] = [] + for child in node.children: + if isinstance(child, str): + parts.append(child) + else: + parts.append(_node_text(child)) + return " ".join(part for part in parts if part).strip() + + +def _clean_label(text: str) -> str: + cleaned = re.sub(r"\s+", " ", text).strip().rstrip(":") + cleaned = re.sub(r"\s*(überspringen|entfernen)$", "", cleaned, flags=re.IGNORECASE).strip() + if not cleaned or _skip_option(cleaned): + return "" + return cleaned + + +def _skip_option(label: str) -> bool: + return label.strip().lower() in SKIP_OPTION_LABELS + + +def _first_number(text: str) -> float | int | None: + match = NUMBER_RE.search(text.replace(",", ".")) + if match is None: + return None + raw = match.group(1) + value = float(raw) + return int(value) if value.is_integer() else value + + +def _unit(text: str) -> str | None: + match = UNIT_RE.search(text.replace(" ", "")) + if match is None: + if "kg" in text.lower(): + return "kg" + return None + unit = match.group(1) + if unit in {'"', "″"}: + return "inch" + return unit + + +def _category_name(root: _Node, fallback: str) -> str: + titles: list[str] = [] + + def walk(node: _Node) -> None: + if node.tag in {"h1", "title"}: + text = _node_text(node) + if text: + titles.append(text) + for child in node.children: + if isinstance(child, _Node): + walk(child) + + walk(root) + if titles: + return titles[0].split("»")[0].strip() or fallback + return "Notebooks" if fallback == "nb" else fallback diff --git a/backend/fixtures/geizhals/nb-category.html b/backend/fixtures/geizhals/nb-category.html new file mode 100644 index 0000000..c7cc3bc --- /dev/null +++ b/backend/fixtures/geizhals/nb-category.html @@ -0,0 +1,132 @@ + + + + Notebooks Preisvergleich Geizhals Deutschland + + +

Notebooks

+
+ RAM + ab 16GB + ab 32GB +
+
+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Hersteller: + Apple + ASUS + Lenovo +
Display-Größe: + 13.x" + 14.x" + 15.x" + 16.x" +
Display-Auflösung: + 1920x1080 (Full HD) + 2560x1600 (WQXGA) +
CPU-Kerne: + 8 (Octa-Core) + 10 + 14-Core +
CPU-Kerne ab: + ab 8 + ab 10 +
CPU-Threads ab: + ab 12 + ab 16 +
CPU-Turbotakt: + ab 4.5GHz + ab 5GHz +
RAM ab: + ab 16GB + ab 32GB + ab 64GB +
RAM-Typ: + DDR5 + LPDDR5 +
Speichertyp: + SSD + M.2 +
SSD-Kapazität: + ab 1TB + ab 2TB +
Grafik-Typ: + dedizierte Grafik (dGPU) +
GPU - Speichergröße ab: + ab 2GB + ab 4GB + ab 8GB +
GPU - Speichergröße: + 8GB + 12GB +
USB-C mit DisplayPort gesamt (inkl. Thunderbolt): + ab 1x +
+ + diff --git a/backend/fixtures/geizhals/nb-search-page.html b/backend/fixtures/geizhals/nb-search-page.html new file mode 100644 index 0000000..4642e74 --- /dev/null +++ b/backend/fixtures/geizhals/nb-search-page.html @@ -0,0 +1,52 @@ + + + + 6431 Produkte + +
+

+ + Acme Notebook 14, 32GB RAM, 1TB SSD + +

+ ACM-14-32 +
+
Betriebssystem
Linux
+
Display
14"
+
CPU
Acme 8-Core
+
RAM
32GB
+
Speicher
1TB SSD
+
Grafik
Acme GPU
+
+
+ + € 1298,00 + + +
+ 8 Angebote +
+
+

+ + Acme Notebook 13, 8GB RAM + +

+ ACM-13-8 +
+
RAM
8GB
+
Speicher
512GB SSD
+
+
+ + € 899,00 + + +
+ 3 Angebote +
+ + diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8e9046c..e346213 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -6,6 +6,7 @@ requires-python = ">=3.13" dependencies = [ "alembic>=1.16", "fastapi>=0.118", + "httpx>=0.28", "psycopg[binary]>=3.2", "pydantic>=2.11", "pydantic-settings>=2.11", @@ -18,6 +19,9 @@ test = [ "httpx>=0.28", "pytest>=8.4", ] +live = [ + "playwright>=1.48", +] [build-system] requires = ["setuptools>=75"] diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 80f8fa7..714493e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,4 +1,5 @@ import json +import os from datetime import datetime, timezone from decimal import Decimal from pathlib import Path @@ -6,6 +7,8 @@ from uuid import UUID, uuid4 import pytest +os.environ.setdefault("GEIZHALS_USE_FIXTURES", "1") + from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType from app.domain.models import ( AttributeDefinition, @@ -17,7 +20,17 @@ from app.domain.models import ( ) from app.domain.validation import parse_profile, parse_scoring_model +from app.settings import get_settings + ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture(autouse=True) +def _geizhals_fixtures(monkeypatch): + monkeypatch.setenv("GEIZHALS_USE_FIXTURES", "1") + get_settings.cache_clear() + yield + get_settings.cache_clear() AS_OF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc) CUTOFF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc) diff --git a/backend/tests/unit/test_discovery_urlgen.py b/backend/tests/unit/test_discovery_urlgen.py index af70bee..9483fb8 100644 --- a/backend/tests/unit/test_discovery_urlgen.py +++ b/backend/tests/unit/test_discovery_urlgen.py @@ -68,6 +68,33 @@ def test_broken_mapping_is_not_used(): assert "ram_gb" in internal +def test_multiple_xf_filters_join_with_tilde(): + url, _ = generate_search_url( + category_url="https://geizhals.de/?cat=nb", + criteria_items=[ + {"criterion": "ram_gb", "operator": ">=", "value": 16}, + {"criterion": "gpu", "operator": ">=", "value": 8}, + ], + mappings=[ + { + "criterion_key": "ram_gb", + "external_filter_key": "12", + "source_value": "16384", + "encoding": {"url_param": "xf"}, + "status": "VERIFIED", + }, + { + "criterion_key": "gpu", + "external_filter_key": "7436", + "source_value": "8192", + "encoding": {"url_param": "xf"}, + "status": "VERIFIED", + }, + ], + ) + assert "xf=12_16384~7436_8192" in url or "xf=7436_8192~12_16384" in url + + def test_roundtrip_verifies_xf_and_maxp(): assert verify_roundtrip( category_url="https://geizhals.de/?cat=nb", diff --git a/backend/tests/unit/test_geizhals_result_parser.py b/backend/tests/unit/test_geizhals_result_parser.py new file mode 100644 index 0000000..f9ebde1 --- /dev/null +++ b/backend/tests/unit/test_geizhals_result_parser.py @@ -0,0 +1,42 @@ +from pathlib import Path + +from app.sources.geizhals.result_parser import parse_result_html, with_page + +HTML = (Path(__file__).resolve().parents[2] / "fixtures" / "geizhals" / "nb-search-page.html").read_text( + encoding="utf-8" +) + + +def test_parser_reads_observed_gallery_cards() -> None: + payload = parse_result_html(HTML, page_url="https://geizhals.de/?cat=nb") + assert payload["blocked"] is False + assert payload["page_count"] == 4 + assert payload["result_count"] == 6431 + assert [item["source_external_id"] for item in payload["items"]] == ["1000001", "1000002"] + first = payload["items"][0] + assert first["source_url"] == "https://geizhals.de/acme-notebook-14-a1000001.html" + assert first["source_title"].startswith("Acme Notebook 14") + assert first["manufacturer"] == "Acme" + assert first["manufacturer_sku"] == "ACM-14-32" + assert first["best_price"] == "1298.00" + assert first["currency"] == "EUR" + assert first["availability"] == "in_stock" + assert first["seller_count"] == 8 + assert first["raw_attributes"]["RAM"] == "32GB" + assert first["raw_attributes"]["Betriebssystem"] == "Linux" + assert first["normalized_attributes"]["ram_gb"] == 32 + assert first["normalized_attributes"]["storage_gb"] == 1024 + + +def test_challenge_html_is_not_turned_into_candidates() -> None: + payload = parse_result_html( + "Enable JavaScript and cookies to continue", + page_url="https://geizhals.de/?cat=nb", + ) + assert payload["blocked"] is True + assert payload["items"] == [] + + +def test_with_page_uses_geizhals_pg_parameter() -> None: + assert with_page("https://geizhals.de/?cat=nb&xf=17023_ab+16GB", 1) == "https://geizhals.de/?cat=nb&xf=17023_ab+16GB" + assert "pg=3" in with_page("https://geizhals.de/?cat=nb&xf=17023_ab+16GB", 3) diff --git a/backend/tests/unit/test_geizhals_taxonomy_parser.py b/backend/tests/unit/test_geizhals_taxonomy_parser.py new file mode 100644 index 0000000..1561736 --- /dev/null +++ b/backend/tests/unit/test_geizhals_taxonomy_parser.py @@ -0,0 +1,76 @@ +from pathlib import Path + +from app.discovery.urlgen import generate_search_url +from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html + +HTML = (Path(__file__).resolve().parents[2] / "fixtures" / "geizhals" / "nb-category.html").read_text( + encoding="utf-8" +) + + +def test_parser_reads_href_filters_and_price_inputs() -> None: + payload = parse_taxonomy_html( + HTML, + category_ref="nb", + source_url="https://geizhals.de/?cat=nb", + ) + keys = {item["external_filter_key"] for item in payload["filters"]} + assert {"12", "17023", "price", "12128", "7436", "82", "asuch"} <= keys + assert len(payload["filters"]) >= 12 + live_ram = next(item for item in payload["filters"] if item["external_filter_key"] == "17023") + assert live_ram["label"] == "RAM" + ram = next(item for item in payload["filters"] if item["external_filter_key"] == "12") + assert ram["label"] == "RAM ab" + assert ram["filter_type"] == "NUMERIC_THRESHOLD" + assert any(option["source_value"] == "ab 16GB" for option in ram["options"]) + assert ram["encoding"]["options"]["ab 16GB"] == "16384" + price = next(item for item in payload["filters"] if item["external_filter_key"] == "price") + assert price["filter_type"] == "NUMERIC_RANGE" + assert price["encoding"]["min_parameter"] == "bpmin" + assert price["encoding"]["max_parameter"] == "bpmax" + + +def test_learned_ram_and_price_encode_observed_geizhals_query() -> None: + payload = parse_taxonomy_html( + HTML, + category_ref="nb", + source_url="https://geizhals.de/?cat=nb", + ) + ram = next(item for item in payload["filters"] if item["external_filter_key"] == "12") + price = next(item for item in payload["filters"] if item["external_filter_key"] == "price") + url, internal = generate_search_url( + category_url="https://geizhals.de/?cat=nb", + criteria_items=[ + {"criterion": "geizhals:nb:12", "operator": ">=", "value": 16}, + {"criterion": "geizhals:nb:price", "operator": "<=", "value": 1700}, + ], + mappings=[ + { + "criterion_key": "geizhals:nb:12", + "external_filter_key": "12", + "source_value": "16384", + "encoding": ram["encoding"], + "status": "VERIFIED", + }, + { + "criterion_key": "geizhals:nb:price", + "external_filter_key": "price", + "source_value": None, + "encoding": price["encoding"], + "status": "VERIFIED", + }, + ], + ) + assert internal == [] + assert "xf=12_16384" in url + assert "bpmax=1700" in url + + +def test_challenge_html_is_not_treated_as_taxonomy() -> None: + payload = parse_taxonomy_html( + "Enable JavaScript and cookies to continue", + category_ref="nb", + source_url="https://geizhals.de/?cat=nb", + ) + assert payload["blocked"] is True + assert payload["filters"] == [] diff --git a/backend/tests/unit/test_settings.py b/backend/tests/unit/test_settings.py index a64eb3c..0adc6ab 100644 --- a/backend/tests/unit/test_settings.py +++ b/backend/tests/unit/test_settings.py @@ -12,3 +12,9 @@ def test_database_url_uses_psycopg_and_explicit_fields() -> None: assert settings.database_url == ( "postgresql+psycopg://buyingagent:secret@postgres:5432/buyingagent_test" ) + + +def test_geizhals_live_defaults_are_off_for_fixtures_flag() -> None: + settings = Settings(geizhals_use_fixtures=True, geizhals_playwright=False) + assert settings.geizhals_use_fixtures is True + assert settings.geizhals_playwright is False diff --git a/backend/uv.lock b/backend/uv.lock index c9fb7d9..ebffc96 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -54,6 +54,7 @@ source = { editable = "." } dependencies = [ { name = "alembic" }, { name = "fastapi" }, + { name = "httpx" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -62,6 +63,9 @@ dependencies = [ ] [package.optional-dependencies] +live = [ + { name = "playwright" }, +] test = [ { name = "httpx" }, { name = "pytest" }, @@ -71,7 +75,9 @@ test = [ requires-dist = [ { name = "alembic", specifier = ">=1.16" }, { name = "fastapi", specifier = ">=0.118" }, + { name = "httpx", specifier = ">=0.28" }, { name = "httpx", marker = "extra == 'test'", specifier = ">=0.28" }, + { name = "playwright", marker = "extra == 'live'", specifier = ">=1.48" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.11" }, { name = "pydantic-settings", specifier = ">=2.11" }, @@ -79,7 +85,7 @@ requires-dist = [ { name = "sqlalchemy", specifier = ">=2.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.37" }, ] -provides-extras = ["test"] +provides-extras = ["test", "live"] [[package]] name = "certifi" @@ -125,6 +131,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/a1/e720a38852366c589e1a46cf570b886507ad2cf591050c203365638baab0/greenlet-3.5.6-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:f96f0e30b5a95c7631b12bfe214cbc90ec8fe8cfa36920596c10514a65743519", size = 294627, upload-time = "2026-09-14T14:24:40.102Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c3/58187858df41354a11e6a55b421e7af9059798abdab3a384cc51b8567c38/greenlet-3.5.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c75116c9de79949de23006e2d9b35ee82874c594fcf5c0311b439acaa14b8441", size = 614356, upload-time = "2026-09-14T15:12:03.399Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b9/3a7e67d5f05c9760b1ad411fa52264bd69cc08e22a2ebfb4018b90628ced/greenlet-3.5.6-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cad5782f93f7f738b62c6527b6f32a60694d924029f299a8b524758cfa53d815", size = 626756, upload-time = "2026-09-14T15:20:44.269Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7c/40400455f5b5a65bb83e94fde66d1be9e5ec518638113f8083ace746c309/greenlet-3.5.6-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a93ee7c6e8fd0f8a83525a51bd777be57ee17787e91d805bd8d6faf9dcada18e", size = 632632, upload-time = "2026-09-14T15:25:07.813Z" }, + { url = "https://files.pythonhosted.org/packages/85/cb/ab0c123c514ed4e94c0dc9ee2e86362633e6b998cfc05de7fc9ac2eb9690/greenlet-3.5.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f98e8215e172f567ce80eeaed9107fb4d32b6c44f26983d9b8334658136a205a", size = 623779, upload-time = "2026-09-14T14:36:01.104Z" }, + { url = "https://files.pythonhosted.org/packages/f9/67/1f35cff30a6c51c3f23b63d4afcc7313ab4f97490ba3676fa78178984b27/greenlet-3.5.6-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7f731ebac68ea06d628658295cb2d217b10186329fcf9a3b6a149045059bf92e", size = 434933, upload-time = "2026-09-14T15:28:38.858Z" }, + { url = "https://files.pythonhosted.org/packages/a5/26/fda8a5a06e7073333ccb038133c5893b9e0c4fe29d5992a17e83c241bc6e/greenlet-3.5.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df19e2d0b1620039af5102563fbd96e8938c7f5c3f5828528d641d9fc585525e", size = 1584930, upload-time = "2026-09-14T15:10:08.234Z" }, + { url = "https://files.pythonhosted.org/packages/2f/37/50f8813163148d6234e08b23dcad6a9e37f01d148c8ec976e4c44ea2d918/greenlet-3.5.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06c0e933290fba8ffe53ead4ae1b8044b0e9754b75cebf381aa2bc3e50d82fac", size = 1647590, upload-time = "2026-09-14T14:35:51.173Z" }, + { url = "https://files.pythonhosted.org/packages/86/da/b7669b09586365654083a62bd0724cf06cb74bd5085a15cdd161271f992f/greenlet-3.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:5b602b4201b965a8354d74e232364a66ff243dd142e350d035f46169bb36e13d", size = 324086, upload-time = "2026-09-14T14:23:48.428Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5d/c9663cfe84a2a9e0aa96f066f5b0594c227ea4c647511e087e2e11d4ac0a/greenlet-3.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:876077e7ebb8c84ed068e2b23d4c62ebb010d60df84b9591af1be2f39010ffb2", size = 308211, upload-time = "2026-09-14T14:28:01.634Z" }, + { url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" }, + { url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" }, + { url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" }, + { url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" }, + { url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" }, + { url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" }, + { url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" }, + { url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" }, + { url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" }, + { url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" }, + { url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" }, + { url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" }, + { url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" }, + { url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" }, + { url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" }, + { url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" }, + { url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" }, + { url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" }, + { url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" }, + { url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" }, + { url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -340,6 +403,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "playwright" +version = "1.63.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/dd/fbb3d34228ad753bc14464d5f2252585b73367539775a88d0a3949cf5a45/playwright-1.63.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:84c540759e8e7f01e690e197e04060f48899d37cea322299f255843273d3385d", size = 44278407, upload-time = "2026-09-15T16:49:06.045Z" }, + { url = "https://files.pythonhosted.org/packages/f5/9a/948b930b1a8c4ee869e5a139a2b7747caa06aab56a3f09a2f0abdcbda221/playwright-1.63.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fd1aa00631d44d55e56e0975bf3f3f285fac4a9fd2813183f0c12a488f1a1b24", size = 42944679, upload-time = "2026-09-15T16:49:10.039Z" }, + { url = "https://files.pythonhosted.org/packages/94/11/dc5c13fa1602371603acd461be47529c1b3513815d3a0dc98f642c291a10/playwright-1.63.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:c89fc4736502a1f0fac2c8ca5d10c0cbc1c669f1f4774a2d8507a43140e4d53f", size = 44278410, upload-time = "2026-09-15T16:49:13.861Z" }, + { url = "https://files.pythonhosted.org/packages/27/9c/103a5037789062bdab27c7dca53f3ca6b075b572ab2cd96eec825b3aec4e/playwright-1.63.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ad21bc07516b187965a7521c5cf0df0bd657b17482eaad74335272d35a2b07de", size = 48217159, upload-time = "2026-09-15T16:49:17.404Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/3d85505284c5a210f2da6c07b8f757524e79d1fba9cfdafe1eafb766ae59/playwright-1.63.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:354e15b29503565fc598b89f16fbe070459343bef9d7498a93e304864000c6a7", size = 47902708, upload-time = "2026-09-15T16:49:21.055Z" }, + { url = "https://files.pythonhosted.org/packages/69/8d/f74ff6b52751859f4b69caf66aa7d4a3c8d6c1f0c7dc9920da103612ee39/playwright-1.63.0-py3-none-win32.whl", hash = "sha256:660c00c62639e31b16700ba5456b351ddba55bb766b7ce8261223aa34928e482", size = 38606075, upload-time = "2026-09-15T16:49:29.546Z" }, + { url = "https://files.pythonhosted.org/packages/76/eb/d6b8d92658038e260dbc7dd69fb3fdbe245aac283953de8b50b02bfe5f61/playwright-1.63.0-py3-none-win_amd64.whl", hash = "sha256:2f9a707a6c6c91157ed77bff2b8caeb04b3c8d46e70d585fc134298cbe4b5cc6", size = 38606082, upload-time = "2026-09-15T16:49:32.838Z" }, + { url = "https://files.pythonhosted.org/packages/be/75/2432b3e3c7c62103b72d4c4cc8b16a56383ada372bbb0d1278591e988f71/playwright-1.63.0-py3-none-win_arm64.whl", hash = "sha256:1e4a3a838ce22fb68ad17193fcd142a19610d9d70fb9966d2239f5dddc0cc05b", size = 34523554, upload-time = "2026-09-15T16:49:36.455Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -491,6 +573,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, ] +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + [[package]] name = "pygments" version = "2.21.0" diff --git a/docker-compose.dev-env.yml b/docker-compose.dev-env.yml index f3a6a1b..1294f2a 100644 --- a/docker-compose.dev-env.yml +++ b/docker-compose.dev-env.yml @@ -33,6 +33,8 @@ services: ENVIRONMENT: "${ENVIRONMENT:-development}" ENGINE_BUILD: "${ENGINE_BUILD:-dev}" CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:5173,http://127.0.0.1:5173,http://192.168.2.49:5174}" + GEIZHALS_USE_FIXTURES: "${GEIZHALS_USE_FIXTURES:-0}" + GEIZHALS_PLAYWRIGHT: "${GEIZHALS_PLAYWRIGHT:-1}" ports: - "${BUYING_AGENT_API_PORT:-8095}:8000" depends_on: @@ -84,6 +86,7 @@ services: ENVIRONMENT: test ENGINE_BUILD: test CLOCK_NOW: "2026-10-03T12:00:00Z" + GEIZHALS_USE_FIXTURES: "1" depends_on: postgres: condition: service_healthy diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 28b7fb0..f6a12a0 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -34,6 +34,7 @@ services: ENVIRONMENT: test ENGINE_BUILD: test CLOCK_NOW: "2026-10-03T12:00:00Z" + GEIZHALS_USE_FIXTURES: "1" depends_on: postgres: condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 23c4e52..50bb12e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,8 @@ services: ENVIRONMENT: "${ENVIRONMENT:-production}" ENGINE_BUILD: "${ENGINE_BUILD:-dev}" CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:4173,http://127.0.0.1:4173}" + GEIZHALS_USE_FIXTURES: "${GEIZHALS_USE_FIXTURES:-0}" + GEIZHALS_PLAYWRIGHT: "${GEIZHALS_PLAYWRIGHT:-1}" ports: - "${BUYING_AGENT_API_BIND:-127.0.0.1}:${BUYING_AGENT_API_PORT:-8000}:8000" depends_on: diff --git a/frontend/nginx.conf b/frontend/nginx.conf index ef13eb6..5882ece 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -8,6 +8,9 @@ server { proxy_pass http://api:8000; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 10s; + proxy_read_timeout 120s; + proxy_send_timeout 120s; } location /health { diff --git a/frontend/src/api.ts b/frontend/src/api.ts index c4d6b35..8dab3a7 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -7,7 +7,7 @@ async function request(path: string, init?: RequestInit): Promise { }); const body = await response.json().catch(() => ({})); if (!response.ok) { - throw new Error(body.message || `HTTP ${response.status}`); + throw new Error(body?.error?.message || body?.message || `HTTP ${response.status}`); } return body as T; } @@ -101,7 +101,12 @@ export type AvailableCriterion = { supports_discovery: boolean; scoring_attribute_key?: string | null; }; -export type AvailableCriteria = { product_category_id: string; source: string; criteria: AvailableCriterion[] }; +export type AvailableCriteria = { + product_category_id: string; + source: string; + taxonomy_status?: string | null; + criteria: AvailableCriterion[]; +}; export type DiscoverySearch = { id: string; status: string; diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index 9e3b63a..275a0e6 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -33,9 +33,10 @@ export function AdminPage() {

Source: {source || "–"}

Kategorie: {category || "–"}

Status: {selected?.taxonomy_status || "NOT_LEARNED"}

+

Gelernte Filter: {filters.data?.filters.length ?? 0}

{discover.isError ?

{(discover.error as Error).message}

: null} {!category ? ( diff --git a/frontend/src/pages/WizardPage.tsx b/frontend/src/pages/WizardPage.tsx index f4da244..93bf698 100644 --- a/frontend/src/pages/WizardPage.tsx +++ b/frontend/src/pages/WizardPage.tsx @@ -18,6 +18,7 @@ export function WizardPage() { const [sourceOverride, setSourceOverride] = useState(null); const [selected, setSelected] = useState([]); const [draft, setDraft] = useState({}); + const [criterionQuery, setCriterionQuery] = useState(""); const sources = useQuery({ queryKey: ["category-sources", categoryId], @@ -63,7 +64,12 @@ export function WizardPage() { const sourceRow = sourceItems.find((item) => item.source === source) || sourceItems[0]; const needsPicker = sourceItems.filter((item) => item.status === "ACTIVE").length > 1; - const visible = useMemo(() => criteria.data?.criteria || [], [criteria.data]); + const visible = useMemo(() => { + const items = criteria.data?.criteria || []; + const query = criterionQuery.trim().toLowerCase(); + if (!query) return items; + return items.filter((item) => item.label.toLowerCase().includes(query)); + }, [criteria.data, criterionQuery]); if (categories.isLoading) return ; if (categories.isError) return ; @@ -84,6 +90,7 @@ export function WizardPage() { setCategoryId(event.target.value); setSourceOverride(null); setSelected([]); + setCriterionQuery(""); }} > @@ -139,10 +146,21 @@ export function WizardPage() {

Quelle: {source}

) : null} + {(criteria.data?.taxonomy_status || sourceRow?.taxonomy_status) === "DEGRADED" ? ( +

Die Taxonomie stammt aus einer aufgezeichneten Kategorieseite. Live-Lernen war blockiert.

+ ) : null} {criteria.isError ? criteria.refetch()} /> : null} {criteria.isLoading ? : null} - {visible.length > 0 ? ( + {(criteria.data?.criteria || []).length > 0 ? (
+

+ {(criteria.data?.criteria || []).length} Kriterien aus der Quelle. Wähle die relevanten aus und setze + Grenzwerte. +

+ {visible.map((item) => { const on = selected.includes(item.criterion_definition_id); return (