Lerne Taxonomie und Trefferliste live von Geizhals, damit Wizard und Kandidaten echte Quelldaten nutzen.
Some checks failed
Deploy Development / deploy (push) Successful in 1m49s
Test Suite / pytest-backend (push) Failing after 32s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-checks (push) Successful in 30s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Lars 2026-10-04 14:53:08 +02:00
parent d784248e28
commit 46857715df
25 changed files with 1614 additions and 27 deletions

View File

@ -18,11 +18,13 @@ RUN uv sync --frozen --no-dev --no-install-project
FROM base AS runtime FROM base AS runtime
COPY . . COPY . .
RUN uv sync --frozen --no-dev \ ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
RUN uv sync --frozen --extra live --no-dev \
&& playwright install --with-deps chromium \
&& sed -i 's/\r$//' /app/scripts/startup.sh \ && sed -i 's/\r$//' /app/scripts/startup.sh \
&& chmod +x /app/scripts/startup.sh \ && chmod +x /app/scripts/startup.sh \
&& useradd --create-home --uid 1000 app \ && useradd --create-home --uid 1000 app \
&& chown -R app:app /app && chown -R app:app /app /ms-playwright
USER app USER app
EXPOSE 8000 EXPOSE 8000

View File

@ -27,11 +27,17 @@ def generate_search_url(
base = urlsplit(category_url) base = urlsplit(category_url)
query = dict(parse_qsl(base.query, keep_blank_values=True)) query = dict(parse_qsl(base.query, keep_blank_values=True))
extra: list[tuple[str, str]] = [] extra: list[tuple[str, str]] = []
xf_values: list[str] = []
for name, value in fragments: for name, value in fragments:
if name in query and name != "xf": if name == "xf":
xf_values.append(value)
continue
if name in query:
query[name] = value query[name] = value
else: else:
extra.append((name, value)) extra.append((name, value))
if xf_values:
extra.append(("xf", "~".join(xf_values)))
encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote) encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote)
url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment)) url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment))
return url, internal_only return url, internal_only
@ -40,11 +46,15 @@ def generate_search_url(
def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]: def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
encoding = mapping.get("encoding") or {} encoding = mapping.get("encoding") or {}
param = str(encoding.get("url_param") or "xf") param = str(encoding.get("url_param") or "xf")
if encoding.get("encoding_type") == "RANGE_PARAMETERS" or param in {"bpmin", "bpmax"}:
return _encode_range(encoding, criterion)
if encoding.get("encoding_type") == "RANGE_TOKEN":
return _encode_range_token(mapping, criterion)
if param == "xf": if param == "xf":
filter_key = str(mapping["external_filter_key"]) filter_key = str(mapping["external_filter_key"])
source_value = str(mapping.get("source_value") or _threshold_value(mapping, criterion)) source_value = _xf_value(mapping, criterion)
return [("xf", f"{filter_key}_{_plus(source_value)}")] return [("xf", f"{filter_key}_{_plus(source_value)}")]
if param in {"minp", "maxp", "prmin", "prmax"}: if param in {"minp", "maxp", "prmin", "prmax", "asuch", "fs"}:
return [(param, str(criterion.get("value")))] return [(param, str(criterion.get("value")))]
template = encoding.get("value_template") template = encoding.get("value_template")
if template: if template:
@ -52,6 +62,63 @@ def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[
raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}") raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}")
def _encode_range(encoding: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
operator = str(criterion.get("operator") or "<=")
value = criterion.get("value")
min_param = str(encoding.get("min_parameter") or encoding.get("url_param") or "bpmin")
max_param = str(encoding.get("max_parameter") or "bpmax")
if operator == "between":
if isinstance(value, dict):
items: list[tuple[str, str]] = []
if value.get("min") is not None:
items.append((min_param, str(value["min"])))
if value.get("max") is not None:
items.append((max_param, str(value["max"])))
return items
if isinstance(value, list) and len(value) == 2:
return [(min_param, str(value[0])), (max_param, str(value[1]))]
if operator in {">=", ">"}:
return [(min_param, str(value))]
return [(max_param, str(value))]
def _encode_range_token(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
filter_key = str(mapping["external_filter_key"])
looked_up = _lookup_option(mapping, criterion.get("value"))
if looked_up is not None:
return [("xf", f"{filter_key}_{_plus(looked_up)}")]
operator = str(criterion.get("operator") or "<=")
value = criterion.get("value")
if operator == "between":
if isinstance(value, dict):
raw = f"{value.get('min') or ''}*{value.get('max') or ''}"
elif isinstance(value, list) and len(value) == 2:
raw = f"{value[0]}*{value[1]}"
else:
raw = f"*{value}"
elif operator in {">=", ">"}:
raw = f"{value}*"
else:
raw = f"*{value}"
return [("xf", f"{filter_key}_{raw}")]
def _lookup_option(mapping: dict[str, Any], value: Any) -> str | None:
options = (mapping.get("encoding") or {}).get("options") or {}
if value in options:
return str(options[value])
if str(value) in options:
return str(options[str(value)])
return None
def _xf_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
looked_up = _lookup_option(mapping, criterion.get("value"))
if looked_up is not None:
return looked_up
return str(mapping.get("source_value") or _threshold_value(mapping, criterion))
def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str: def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
option = mapping.get("source_value") option = mapping.get("source_value")
if option: if option:
@ -72,8 +139,12 @@ def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]:
parts = urlsplit(url) parts = urlsplit(url)
decoded: list[tuple[str, str]] = [] decoded: list[tuple[str, str]] = []
for name, value in parse_qsl(parts.query, keep_blank_values=True): for name, value in parse_qsl(parts.query, keep_blank_values=True):
if name == "xf" and "_" in value: if name != "xf":
key, raw = value.split("_", 1) continue
for part in value.replace("|", "~").split("~"):
if "_" not in part:
continue
key, raw = part.split("_", 1)
decoded.append((key, raw.replace("+", " "))) decoded.append((key, raw.replace("+", " ")))
return decoded return decoded

View File

@ -100,7 +100,7 @@ class CatalogService:
SourceCategoryRecord.product_category_id == product_category_id, SourceCategoryRecord.product_category_id == product_category_id,
) )
).first() ).first()
if source_category is not None and source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value: if source_category is not None and self._should_learn(source_category, source_id):
from app.services.discovery_service import TaxonomyService from app.services.discovery_service import TaxonomyService
TaxonomyService(self.session).discover(source_id, source_category.external_category_key) TaxonomyService(self.session).discover(source_id, source_category.external_category_key)
@ -172,7 +172,33 @@ class CatalogService:
if definition.id in seen: if definition.id in seen:
continue continue
criteria.append(self._criterion_out(definition, None, None, [], None)) criteria.append(self._criterion_out(definition, None, None, [], None))
return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria}) return jsonable(
{
"product_category_id": product_category_id,
"source": source_id,
"taxonomy_status": None if source_category is None else source_category.taxonomy_status,
"criteria": criteria,
}
)
def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool:
if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
return True
if source_category.taxonomy_status != TaxonomyStatus.READY.value:
return False
from app.settings import get_settings
if get_settings().geizhals_use_fixtures or source_id != "geizhals":
return False
keys = {
row.external_filter_key
for row in self.session.scalars(
select(SourceFilterDefinitionRecord).where(
SourceFilterDefinitionRecord.source_category_id == source_category.id
)
)
}
return keys <= {"17023", "price"}
def list_templates(self) -> list[dict[str, Any]]: def list_templates(self) -> list[dict[str, Any]]:
rows, _total = self.catalog.list_profiles(200, 0) rows, _total = self.catalog.list_profiles(200, 0)

View File

@ -14,6 +14,7 @@ from app.discovery.diff import diff_run, next_lifecycle
from app.discovery.enums import ( from app.discovery.enums import (
CandidateStatus, CandidateStatus,
DiffStatus, DiffStatus,
FilterType,
LifecycleStatus, LifecycleStatus,
MappingStatus, MappingStatus,
MatchStatus, MatchStatus,
@ -99,7 +100,20 @@ class TaxonomyService:
if catalog.get_source(source_id) is None: if catalog.get_source(source_id) is None:
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
adapter = self.adapter or discovery_adapter(source_id) adapter = self.adapter or discovery_adapter(source_id)
payload = adapter.discover_taxonomy(category_key) try:
payload = adapter.discover_taxonomy(category_key)
except AppError as exc:
category = self.session.scalars(
select(SourceCategoryRecord).where(
SourceCategoryRecord.source_id == source_id,
SourceCategoryRecord.external_category_key == category_key,
)
).first()
if category is not None and exc.code == "SOURCE_BLOCKED":
category.taxonomy_status = TaxonomyStatus.BLOCKED.value
category.last_taxonomy_scan_at = now_utc()
self.session.flush()
raise
now = now_utc() now = now_utc()
category = self.session.scalars( category = self.session.scalars(
select(SourceCategoryRecord).where( select(SourceCategoryRecord).where(
@ -148,7 +162,7 @@ class TaxonomyService:
label=str(raw.get("label") or raw["external_filter_key"]), label=str(raw.get("label") or raw["external_filter_key"]),
filter_type=filter_type, filter_type=filter_type,
canonical_unit=raw.get("canonical_unit"), canonical_unit=raw.get("canonical_unit"),
status=MappingStatus.VERIFIED.value, status=MappingStatus.LEARNED_UNVERIFIED.value,
) )
self.session.add(filt) self.session.add(filt)
self.session.flush() self.session.flush()
@ -161,9 +175,14 @@ class TaxonomyService:
self._sync_options(filt, raw.get("options") or []) self._sync_options(filt, raw.get("options") or [])
if raw.get("range"): if raw.get("range"):
self._sync_range(filt, raw["range"]) self._sync_range(filt, raw["range"])
category.taxonomy_status = ( self._sync_learned_mapping(category, filt, raw)
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value requested = str(payload.get("taxonomy_status") or "")
) if requested in {item.value for item in TaxonomyStatus}:
category.taxonomy_status = requested
else:
category.taxonomy_status = (
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
)
self.session.flush() self.session.flush()
return self.filters(source_id, category.external_category_key) return self.filters(source_id, category.external_category_key)
@ -275,6 +294,89 @@ class TaxonomyService:
if removed: if removed:
self._break_mappings(filt.source_category_id, filt.external_filter_key) self._break_mappings(filt.source_category_id, filt.external_filter_key)
def _sync_learned_mapping(
self,
category: SourceCategoryRecord,
filt: SourceFilterDefinitionRecord,
raw: dict[str, Any],
) -> None:
encoding = raw.get("encoding") or {"url_param": "xf"}
definition = upsert_definition(
self.session,
{
"key": f"{category.source_id}:{category.external_category_key}:{filt.external_filter_key}",
"origin": "SOURCE",
"source": category.source_id,
"product_category_id": category.product_category_id,
"label": filt.label,
"value_type": "number"
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
else "string",
"unit": filt.canonical_unit,
"category": category.product_category_id,
},
)
mapping = self.session.scalars(
select(CriterionMappingRecord).where(
CriterionMappingRecord.source_category_id == category.id,
CriterionMappingRecord.external_filter_key == filt.external_filter_key,
)
).first()
options = raw.get("options") or []
source_value = None
if options:
source_value = str(options[0].get("query_value") or options[0].get("source_value"))
now = now_utc()
if mapping is None:
mapping = CriterionMappingRecord(
id=uuid4(),
criterion_definition_id=definition.id,
criterion_key=definition.key,
source_id=category.source_id,
source_category_id=category.id,
external_filter_key=filt.external_filter_key,
source_value=source_value,
encoding=encoding,
status=MappingStatus.LEARNED_UNVERIFIED.value,
definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})),
created_at=now,
updated_at=now,
)
self.session.add(mapping)
self.session.flush()
else:
mapping.criterion_definition_id = definition.id
mapping.criterion_key = definition.key
mapping.source_value = source_value
mapping.encoding = encoding
mapping.definition_hash = sha256_text(canonical_json({"encoding": encoding, "source_value": source_value}))
mapping.updated_at = now
probe_value = source_value or 0
if options and options[0].get("normalized_value") is not None:
probe_value = options[0]["normalized_value"]
operator = (options[0].get("operator") if options else None) or "<="
if filt.filter_type == FilterType.NUMERIC_RANGE.value:
operator = ">="
probe_value = 1000
verified = verify_roundtrip(
category_url=category.source_url or f"https://geizhals.de/?cat={category.external_category_key}",
criterion={
"criterion": definition.key,
"operator": operator,
"value": probe_value,
},
mapping={
"criterion_key": definition.key,
"external_filter_key": filt.external_filter_key,
"source_value": source_value,
"encoding": encoding,
},
)
if mapping.status != MappingStatus.DISABLED.value and verified:
mapping.status = MappingStatus.VERIFIED.value
elif mapping.status == MappingStatus.BROKEN.value:
mapping.status = MappingStatus.LEARNED_UNVERIFIED.value
def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None: def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None:
row = self.session.get(SourceFilterRangeRecord, filt.id) row = self.session.get(SourceFilterRangeRecord, filt.id)
if row is None: if row is None:

View File

@ -16,6 +16,8 @@ class Settings(BaseSettings):
engine_build: str = "dev" engine_build: str = "dev"
clock_now: datetime | None = None clock_now: datetime | None = None
cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173" cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173"
geizhals_use_fixtures: bool = False
geizhals_playwright: bool = True
@property @property
def cors_origins(self) -> list[str]: def cors_origins(self) -> list[str]:

View File

@ -7,6 +7,10 @@ from urllib.parse import parse_qs, urlparse
from app.discovery.enums import FilterType from app.discovery.enums import FilterType
from app.errors import AppError from app.errors import AppError
from app.settings import get_settings
from app.sources.geizhals.live_page import load_category_page, load_result_page
from app.sources.geizhals.result_parser import parse_result_html, with_page
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals" FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
@ -14,12 +18,31 @@ FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
class GeizhalsAdapter: class GeizhalsAdapter:
source_key = "geizhals" source_key = "geizhals"
def __init__(self, fixture_dir: Path | None = None, live: bool = False) -> None: def __init__(self, fixture_dir: Path | None = None, live: bool | None = None) -> None:
self.fixture_dir = fixture_dir or FIXTURE_DIR self.fixture_dir = fixture_dir or FIXTURE_DIR
self.live = live settings = get_settings()
self.live = (not settings.geizhals_use_fixtures) if live is None else live
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]: def discover_taxonomy(self, category_ref: str) -> dict[str, Any]:
payload = self._load(f"{category_ref}-taxonomy.json") if not self.live:
return self._load(f"{category_ref}-taxonomy.json")
url = f"https://geizhals.de/?cat={category_ref}&promode=true"
html, mode, status = load_category_page(
url,
category_ref=category_ref,
playwright=get_settings().geizhals_playwright,
)
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=url)
if payload.get("blocked") or not payload.get("filters"):
raise AppError(
422,
"TAXONOMY_EMPTY",
"Geizhals-Taxonomie konnte nicht aus der Kategorieseite gelesen werden",
{"url": url, "http_status": status, "mode": mode},
)
payload["discovery_mode"] = mode
payload["http_status"] = status
payload["taxonomy_status"] = "DEGRADED" if mode == "recorded_html" else "READY"
return payload return payload
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
@ -34,13 +57,32 @@ class GeizhalsAdapter:
return {"url": url, "internal_only": internal_only} return {"url": url, "internal_only": internal_only}
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]: def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]:
if self.live:
raise AppError(422, "LIVE_FETCH_DISABLED", "live Geizhals fetch is not enabled in this process")
page = int(pagination.get("page") or 1) page = int(pagination.get("page") or 1)
category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0] if not self.live:
return self._load(f"{category}-search-page{page}.json") category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0]
return self._load(f"{category}-search-page{page}.json")
url = with_page(str(query["url"]), page)
html, mode, status, blocked = load_result_page(url, playwright=get_settings().geizhals_playwright)
if blocked:
return {
"page": page,
"page_count": page,
"result_count": 0,
"http_status": status or 403,
"blocked": True,
"challenge": "captcha" if status in {401, 403} else None,
"items": [],
"discovery_mode": mode,
}
payload = parse_result_html(html, page_url=url)
payload["page"] = page
payload["http_status"] = status
payload["discovery_mode"] = mode
return payload
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]: def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]:
if response.get("html") and not response.get("items"):
return list(parse_result_html(str(response["html"]), page_url=str(response.get("url") or "")).get("items") or [])
return list(response.get("items") or []) return list(response.get("items") or [])
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]: def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]:

View File

@ -0,0 +1,104 @@
from __future__ import annotations
from pathlib import Path
from app.errors import AppError
from app.sources.geizhals.taxonomy_parser import is_challenge_html
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
BROWSER_HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
}
EXPAND_LABELS = (
"Immer alle Filter anzeigen",
"weitere Filter anzeigen",
)
def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, int, bool]:
status, html = 0, ""
try:
status, html = _fetch_http(url)
if not _blocked(status, html):
return html, "live_http", status, False
except Exception:
status, html = 0, ""
if playwright:
try:
status, html = _fetch_playwright(url)
if not _blocked(status, html):
return html, "live_playwright", status, False
except Exception:
pass
if html and not is_challenge_html(html) and status and status < 400:
return html, "live_http", status, False
return html, "blocked", status or 403, True
def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]:
status, html = 0, ""
try:
status, html = _fetch_http(url)
if not _blocked(status, html):
return html, "live_http", status
except Exception:
status, html = 0, ""
if playwright:
try:
status, html = _fetch_playwright(url)
if not _blocked(status, html):
return html, "live_playwright", status
except Exception:
pass
recorded = FIXTURE_DIR / f"{category_ref}-category.html"
if recorded.exists():
return recorded.read_text(encoding="utf-8"), "recorded_html", 0
raise AppError(
423,
"SOURCE_BLOCKED",
"Geizhals blockiert den Taxonomie-Abruf (403/Challenge)",
{"url": url, "http_status": status},
)
def _fetch_http(url: str) -> tuple[int, str]:
import httpx
response = httpx.get(url, headers=BROWSER_HEADERS, follow_redirects=True, timeout=25.0)
return response.status_code, response.text
def _fetch_playwright(url: str) -> tuple[int, str]:
try:
from playwright.sync_api import sync_playwright
except ImportError as exc:
raise AppError(
423,
"SOURCE_BLOCKED",
"Geizhals verlangt einen Browser-Abruf, Playwright ist nicht installiert",
) from exc
with sync_playwright() as playwright:
browser = playwright.chromium.launch(headless=True)
page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS)
response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
page.wait_for_timeout(4000)
for label in EXPAND_LABELS:
button = page.get_by_text(label, exact=False)
if button.count():
button.first.click(timeout=5000)
page.wait_for_timeout(1500)
html = page.content()
status = response.status if response is not None else 200
browser.close()
return status, html
def _blocked(status: int, html: str) -> bool:
if status in {401, 403, 429}:
return True
return is_challenge_html(html)

View File

@ -0,0 +1,285 @@
from __future__ import annotations
import re
from typing import Any
from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit
from app.sources.geizhals.taxonomy_parser import is_challenge_html, node_text, parse_html_tree
PRODUCT_ID_RE = re.compile(r"-a(\d+)\.html", re.IGNORECASE)
PRICE_RE = re.compile(r"(\d{1,3}(?:\.\d{3})+|\d+),(\d{2})")
OFFER_RE = re.compile(r"(\d+)\s*Angebote?", re.IGNORECASE)
PRODUCT_COUNT_RE = re.compile(r"(\d[\d.]*)\s*Produkte", re.IGNORECASE)
CAPACITY_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*(TB|GB)", re.IGNORECASE)
WEIGHT_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*kg", re.IGNORECASE)
AVAILABILITY_BY_CODE = {"0": "out_of_stock", "1": "limited", "2": "in_stock"}
AVAILABILITY_BY_LABEL = {
"lagernd": "in_stock",
"kurzfristig lieferbar": "limited",
"kurzfristig": "limited",
"nicht verfügbar": "out_of_stock",
}
def parse_result_html(html: str, *, page_url: str) -> dict[str, Any]:
if is_challenge_html(html):
return {"items": [], "page_count": 1, "result_count": 0, "blocked": True}
root = parse_html_tree(html)
cards = _find_cards(root)
items: list[dict[str, Any]] = []
seen: set[str] = set()
for card in cards:
item = _parse_card(card, page_url)
if item is None or item["source_external_id"] in seen:
continue
seen.add(item["source_external_id"])
items.append(item)
page_count = _page_count(root)
result_count = _result_count(root) or len(items)
return {
"items": items,
"page_count": page_count,
"result_count": result_count,
"blocked": False,
}
def with_page(url: str, page: int) -> str:
parts = urlsplit(url)
query = [(name, value) for name, value in parse_qsl(parts.query, keep_blank_values=True) if name != "pg"]
if page > 1:
query.append(("pg", str(page)))
return parts._replace(query=urlencode(query, doseq=True)).geturl()
def _find_cards(root: Any) -> list[Any]:
cards: list[Any] = []
def walk(node: Any) -> None:
classes = node.attrs.get("class", "")
if "galleryview__item" in classes or "listview__item" in classes:
cards.append(node)
return
for child in node.children:
if not isinstance(child, str):
walk(child)
walk(root)
return cards
def _parse_card(card: Any, page_url: str) -> dict[str, Any] | None:
link = _first_link(card, "name-link") or _first_product_link(card)
if link is None:
return None
href = link.attrs.get("href") or ""
match = PRODUCT_ID_RE.search(href)
if match is None:
return None
title = (link.attrs.get("title") or node_text(link)).strip()
if not title:
return None
source_url = urljoin(page_url, href.split("#")[0])
raw_attributes = _definition_list(card)
sku = _first_text_by_class(card, "galleryview__mpn") or _first_text_by_class(card, "listview__mpn")
price_text = _first_text_by_class(card, "price")
offers_text = _first_text_by_class(card, "offercount") or ""
availability_node = _first_by_attr(card, "data-availability")
manufacturer, model = _split_title(title)
return {
"source_external_id": match.group(1),
"source_url": source_url,
"source_title": re.sub(r"\s+", " ", title),
"manufacturer": manufacturer,
"model": model,
"manufacturer_sku": sku,
"gtin_ean": None,
"best_price": _parse_price(price_text),
"currency": "EUR" if price_text and "€" in price_text else None,
"availability": _availability(availability_node),
"seller_count": _seller_count(offers_text),
"raw_attributes": raw_attributes,
"normalized_attributes": _normalize_attributes(raw_attributes),
}
def _first_link(node: Any, fragment: str) -> Any | None:
classes = node.attrs.get("class", "")
if node.tag == "a" and fragment in classes:
return node
for child in node.children:
if not isinstance(child, str):
found = _first_link(child, fragment)
if found is not None:
return found
return None
def _first_product_link(node: Any) -> Any | None:
if node.tag == "a" and PRODUCT_ID_RE.search(node.attrs.get("href") or ""):
href = node.attrs.get("href") or ""
if "#ratings" not in href and "#offerlist" not in href:
return node
for child in node.children:
if not isinstance(child, str):
found = _first_product_link(child)
if found is not None:
return found
return None
def _first_text_by_class(node: Any, fragment: str) -> str | None:
found = _first_by_class(node, fragment)
if found is None:
return None
text = node_text(found).strip()
return re.sub(r"\s+", " ", text) or None
def _first_by_class(node: Any, fragment: str) -> Any | None:
if fragment in node.attrs.get("class", ""):
return node
for child in node.children:
if not isinstance(child, str):
found = _first_by_class(child, fragment)
if found is not None:
return found
return None
def _first_by_attr(node: Any, name: str) -> Any | None:
if name in node.attrs:
return node
for child in node.children:
if not isinstance(child, str):
found = _first_by_attr(child, name)
if found is not None:
return found
return None
def _definition_list(node: Any) -> dict[str, str]:
pairs: dict[str, str] = {}
def walk(current: Any) -> None:
if current.tag == "dl":
pending: str | None = None
for child in current.children:
if isinstance(child, str):
continue
if child.tag == "dt":
pending = re.sub(r"\s+", " ", node_text(child)).strip()
elif child.tag == "dd" and pending:
pairs[pending] = re.sub(r"\s+", " ", node_text(child)).strip()
pending = None
return
for child in current.children:
if not isinstance(child, str):
walk(child)
walk(node)
return pairs
def _split_title(title: str) -> tuple[str | None, str | None]:
parts = title.split(None, 1)
if not parts:
return None, None
if len(parts) == 1:
return parts[0], None
return parts[0], parts[1]
def _parse_price(text: str | None) -> str | None:
if not text:
return None
match = PRICE_RE.search(text.replace("\xa0", " "))
if match is None:
return None
whole = match.group(1).replace(".", "")
return f"{whole}.{match.group(2)}"
def _seller_count(text: str) -> int | None:
match = OFFER_RE.search(text)
if match is None:
return None
return int(match.group(1))
def _availability(node: Any | None) -> str:
if node is None:
return "unknown"
code = AVAILABILITY_BY_CODE.get(node.attrs.get("data-availability", ""))
if code:
return code
label = (node.attrs.get("aria-label") or node_text(node)).strip().casefold()
return AVAILABILITY_BY_LABEL.get(label, "unknown")
def _normalize_attributes(raw: dict[str, str]) -> dict[str, Any]:
normalized: dict[str, Any] = {}
for label, value in raw.items():
key = label.casefold()
capacity = _capacity_gb(value)
if capacity is not None and key in {"ram", "arbeitsspeicher"}:
normalized["ram_gb"] = capacity
elif capacity is not None and key in {"speicher", "ssd", "festplatte"}:
normalized["storage_gb"] = capacity
weight = _weight_kg(value)
if weight is not None and "gewicht" in key:
normalized["weight_kg"] = weight
return normalized
def _capacity_gb(text: str) -> float | int | None:
match = CAPACITY_RE.search(text)
if match is None:
return None
value = float(match.group(1).replace(",", "."))
if match.group(2).upper() == "TB":
value *= 1024
return int(value) if value.is_integer() else value
def _weight_kg(text: str) -> float | None:
match = WEIGHT_RE.search(text)
if match is None:
return None
return float(match.group(1).replace(",", "."))
def _page_count(root: Any) -> int:
pages = {1}
def walk(node: Any) -> None:
href = node.attrs.get("href") or ""
if "pg=" in href and ("pagination" in node.attrs.get("class", "") or _in_pagination(node)):
for name, value in parse_qsl(urlsplit(href).query):
if name == "pg" and value.isdigit():
pages.add(int(value))
for child in node.children:
if not isinstance(child, str):
walk(child)
walk(root)
return max(pages)
def _in_pagination(node: Any) -> bool:
current = node
while current is not None:
classes = current.attrs.get("class", "")
ident = current.attrs.get("id", "")
if "pagination" in classes or ident == "pagination":
return True
current = current.parent
return False
def _result_count(root: Any) -> int | None:
counts = [
int(match.replace(".", ""))
for match in PRODUCT_COUNT_RE.findall(node_text(root))
]
return max(counts) if counts else None

View File

@ -0,0 +1,474 @@
from __future__ import annotations
import re
from html.parser import HTMLParser
from typing import Any
from urllib.parse import parse_qsl, unquote, urlparse
from app.discovery.enums import FilterType
SKIP_OPTION_LABELS = {
"alle",
"entfernen",
"überspringen",
"aktualisieren",
"abonnieren",
"filter zurücksetzen",
"nur beliebteste filter anzeigen",
"immer alle filter anzeigen",
"filter überspringen",
"einfache filter",
"erweiterte filter",
}
RANGE_MIN_NAMES = {"bpmin", "minp", "prmin"}
RANGE_MAX_NAMES = {"bpmax", "maxp", "prmax"}
TEXT_NAMES = {"asuch", "fs"}
HEADING_TAGS = {"th", "h2", "h3", "h4", "h5", "legend", "dt", "caption", "label", "strong"}
NUMBER_RE = re.compile(r"(-?\d+(?:[.,]\d+)?)")
THRESHOLD_RE = re.compile(r"^(ab|bis)\s+", re.IGNORECASE)
UNIT_RE = re.compile(r"(GB|TB|GHz|MHz|Hz|cd/m²|ppi|kg|\"|″|mm|%|€|EUR)$", re.IGNORECASE)
TYPE_MAP = {
"text": FilterType.ENUM.value,
"color": FilterType.ENUM.value,
"text_range": FilterType.NUMERIC_THRESHOLD.value,
"numeric_range": FilterType.NUMERIC_RANGE.value,
}
class _Node:
def __init__(self, tag: str, attrs: dict[str, str]) -> None:
self.tag = tag
self.attrs = attrs
self.children: list[_Node | str] = []
self.parent: _Node | None = None
class _TreeBuilder(HTMLParser):
VOID = {
"area",
"base",
"br",
"col",
"embed",
"hr",
"img",
"input",
"link",
"meta",
"param",
"source",
"track",
"wbr",
}
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.root = _Node("document", {})
self._stack = [self.root]
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
node = _Node(tag, {key: value or "" for key, value in attrs})
parent = self._stack[-1]
node.parent = parent
parent.children.append(node)
if tag not in self.VOID:
self._stack.append(node)
def handle_endtag(self, tag: str) -> None:
for index in range(len(self._stack) - 1, 0, -1):
if self._stack[index].tag == tag:
del self._stack[index:]
break
def handle_data(self, data: str) -> None:
text = data.strip()
if text:
self._stack[-1].children.append(text)
def parse_html_tree(html: str) -> _Node:
builder = _TreeBuilder()
builder.feed(html)
return builder.root
def node_text(node: _Node) -> str:
return _node_text(node)
def parse_taxonomy_html(html: str, *, category_ref: str, source_url: str) -> dict[str, Any]:
if is_challenge_html(html):
return {
"source": "geizhals",
"external_category_key": category_ref,
"name": category_ref,
"source_url": source_url,
"filters": [],
"blocked": True,
}
builder = _TreeBuilder()
builder.feed(html)
groups: dict[str, dict[str, Any]] = {}
_collect_declared_filters(builder.root, groups)
_collect_xf_links(builder.root, groups)
_collect_named_inputs(builder.root, groups)
filters = [
group
for group in groups.values()
if _usable_group(group)
]
filters.sort(key=lambda item: (item["label"].lower(), item["external_filter_key"]))
return {
"source": "geizhals",
"external_category_key": category_ref,
"name": _category_name(builder.root, category_ref),
"canonical_category": "notebook" if category_ref == "nb" else None,
"source_url": source_url,
"filters": filters,
"blocked": False,
}
def is_challenge_html(html: str) -> bool:
sample = html.lower()
markers = (
"cf-mitigated",
"cf-challenge",
"just a moment",
"enable javascript and cookies",
"checking your browser",
"verification successful. waiting for geizhals",
)
return any(marker in sample for marker in markers)
def _collect_declared_filters(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
filter_id = node.attrs.get("data-filter-id")
if filter_id:
source_type = node.attrs.get("data-filter-type") or ""
label = _declared_label(node) or f"Filter {filter_id}"
group = groups.setdefault(filter_id, _empty_group(filter_id, label, source_type))
if label and group["label"].startswith("Filter "):
group["label"] = label
if source_type:
group["filter_type"] = TYPE_MAP.get(source_type, group["filter_type"])
if source_type == "numeric_range":
group["encoding"]["encoding_type"] = "RANGE_TOKEN"
group["encoding"]["value_template"] = "{min}*{max}"
group.setdefault("range", {"minimum": None, "maximum": None, "step": None, "unit": group.get("canonical_unit")})
_collect_xf_links(node, groups, required_key=filter_id, allow_histogram=source_type == "numeric_range")
return
for child in node.children:
if isinstance(child, _Node):
_collect_declared_filters(child, groups)
def _collect_xf_links(
node: _Node,
groups: dict[str, dict[str, Any]],
required_key: str | None = None,
allow_histogram: bool = False,
) -> None:
if node.tag == "a":
href = node.attrs.get("href") or ""
label = node.attrs.get("aria-label") or _node_text(node)
if _usable_option(node, label, allow_histogram=allow_histogram):
for key, query_value in _xf_tokens(href):
if required_key is not None and key != required_key:
continue
if required_key is None and not _has_structural_label(node):
continue
group_label = _declared_label(_filter_root(node)) or _nearest_group_label(node)
group = groups.setdefault(key, _empty_group(key, group_label, None))
if group_label and (group["label"].startswith("Filter ") or group["label"] == key):
group["label"] = group_label
option = _option_from_link(label, query_value)
existing = {item["source_value"] for item in group["options"]}
if option["source_value"] not in existing:
group["options"].append(option)
group["encoding"].setdefault("options", {})
group["encoding"]["options"][option["source_value"]] = query_value
if option.get("normalized_value") is not None:
group["encoding"]["options"][str(option["normalized_value"])] = query_value
if group["filter_type"] not in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}:
group["filter_type"] = _classify_options(group["options"])
if group["canonical_unit"] is None:
group["canonical_unit"] = option.get("unit")
for child in node.children:
if isinstance(child, _Node):
_collect_xf_links(child, groups, required_key=required_key, allow_histogram=allow_histogram)
def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
inputs: list[_Node] = []
_find_inputs(node, inputs)
names = {item.attrs.get("name", ""): item for item in inputs if item.attrs.get("name")}
if names.keys() & RANGE_MIN_NAMES or names.keys() & RANGE_MAX_NAMES:
min_name = next((name for name in RANGE_MIN_NAMES if name in names), "bpmin")
max_name = next((name for name in RANGE_MAX_NAMES if name in names), "bpmax")
label = _nearest_group_label(names.get(min_name) or names.get(max_name) or node) or "Preis"
groups["price"] = {
"external_filter_key": "price",
"label": "Preis" if "preis" not in label.lower() else label,
"filter_type": FilterType.NUMERIC_RANGE.value,
"canonical_unit": "EUR",
"options": [],
"range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"},
"encoding": {
"url_param": min_name,
"min_parameter": min_name,
"max_parameter": max_name,
"encoding_type": "RANGE_PARAMETERS",
},
}
for name in TEXT_NAMES:
if name not in names:
continue
label = _nearest_group_label(names[name]) or "Suchbegriffe"
groups[name] = {
"external_filter_key": name,
"label": label,
"filter_type": FilterType.TEXT.value,
"canonical_unit": None,
"options": [],
"encoding": {"url_param": name, "encoding_type": "TEXT_PARAMETER", "value_template": "{value}"},
}
def _find_inputs(node: _Node, found: list[_Node]) -> None:
if node.tag == "input":
found.append(node)
for child in node.children:
if isinstance(child, _Node):
_find_inputs(child, found)
def _xf_tokens(href: str) -> list[tuple[str, str]]:
if not href or href.startswith("#") or href.startswith("javascript:"):
return []
query = urlparse(href).query
tokens: list[tuple[str, str]] = []
for name, value in parse_qsl(query, keep_blank_values=True):
if name != "xf" or not value:
continue
for part in unquote(value).replace("|", "~").split("~"):
if "_" not in part:
continue
key, raw = part.split("_", 1)
if key.isdigit() and raw:
tokens.append((key, raw.replace("+", " ")))
return tokens
def _option_from_link(label: str, query_value: str) -> dict[str, Any]:
cleaned = re.sub(r"\(\d[\d.]*\s*(produkte)?\)\s*$", "", label, flags=re.IGNORECASE).strip()
cleaned = re.sub(r"\s+", " ", cleaned)
operator = None
if cleaned.lower().startswith("ab "):
operator = ">="
elif cleaned.lower().startswith("bis "):
operator = "<="
number = _first_number(cleaned)
unit = _unit(cleaned)
return {
"source_value": cleaned or query_value,
"query_value": query_value,
"normalized_value": number,
"operator": operator,
"unit": unit,
}
def _classify_options(options: list[dict[str, Any]]) -> str:
if not options:
return FilterType.UNKNOWN.value
labels = [str(item["source_value"]).lower() for item in options]
if all(item in {"ja", "nein", "egal", "true", "false"} for item in labels):
return FilterType.BOOLEAN.value
if all(THRESHOLD_RE.match(str(item["source_value"])) for item in options):
return FilterType.NUMERIC_THRESHOLD.value
return FilterType.ENUM.value
def _empty_group(key: str, label: str | None, source_type: str | None) -> dict[str, Any]:
encoding_type = "OPTION_TOKEN"
if source_type == "numeric_range":
encoding_type = "RANGE_TOKEN"
return {
"external_filter_key": key,
"label": label or f"Filter {key}",
"filter_type": TYPE_MAP.get(source_type or "", FilterType.ENUM.value),
"canonical_unit": None,
"options": [],
"encoding": {
"url_param": "xf",
"encoding_type": encoding_type,
"options": {},
**({"value_template": "{min}*{max}"} if encoding_type == "RANGE_TOKEN" else {}),
},
}
def _declared_label(node: _Node | None) -> str | None:
if node is None:
return None
attr_id = node.attrs.get("id", "")
if attr_id.startswith("filter-element__title-label"):
return _clean_label(_node_text(node)) or None
for child in node.children:
if isinstance(child, _Node):
found = _declared_label(child)
if found:
return found
return None
def _filter_root(node: _Node) -> _Node | None:
current: _Node | None = node
while current is not None:
if current.attrs.get("data-filter-id"):
return current
current = current.parent
return None
def _nearest_group_label(node: _Node | None) -> str | None:
current = node
while current is not None:
declared = _declared_label(current)
if declared:
return declared
if current.tag in HEADING_TAGS:
text = _clean_label(_node_text(current))
if text:
return text
previous = _previous_element(current)
while previous is not None:
if previous.tag in HEADING_TAGS:
text = _clean_label(_node_text(previous))
if text:
return text
previous = _previous_element(previous)
labeled = current.attrs.get("aria-label") or current.attrs.get("data-filter-name") or current.attrs.get("title")
if labeled:
text = _clean_label(labeled)
if text and len(text.split()) <= 6:
return text
current = current.parent
return None
def _has_structural_label(node: _Node) -> bool:
if _filter_root(node) is not None:
return True
current: _Node | None = node
while current is not None:
if current.tag in {"th", "td", "tr", "table", "dt", "dd"}:
return _nearest_group_label(node) is not None
current = current.parent
return False
def _usable_option(node: _Node, label: str, *, allow_histogram: bool) -> bool:
classes = f"{node.attrs.get('class', '')} {node.attrs.get('id', '')}".lower()
if "skip-link" in classes or "add-to-columns" in classes:
return False
if "bucket" in classes and not allow_histogram:
return False
cleaned = re.sub(r"\s+", " ", label).strip()
if not cleaned or _skip_option(cleaned):
return False
if "überspringen" in cleaned.lower():
return False
if not allow_histogram and "produkte" in cleaned.lower():
return False
if len(cleaned.split()) > 8:
return False
return True
def _usable_group(group: dict[str, Any]) -> bool:
if group["filter_type"] == FilterType.TEXT.value:
return True
if group.get("range") and group["filter_type"] == FilterType.NUMERIC_RANGE.value:
return True
if not group.get("options") and not group.get("range"):
return False
label = str(group.get("label") or "")
if label.startswith("Filter ") and len(group.get("options") or []) < 2:
return False
if len(label.split()) > 8:
return False
return True
def _previous_element(node: _Node) -> _Node | None:
if node.parent is None:
return None
siblings = [child for child in node.parent.children if isinstance(child, _Node)]
try:
index = siblings.index(node)
except ValueError:
return None
return siblings[index - 1] if index > 0 else None
def _node_text(node: _Node) -> str:
parts: list[str] = []
for child in node.children:
if isinstance(child, str):
parts.append(child)
else:
parts.append(_node_text(child))
return " ".join(part for part in parts if part).strip()
def _clean_label(text: str) -> str:
cleaned = re.sub(r"\s+", " ", text).strip().rstrip(":")
cleaned = re.sub(r"\s*(überspringen|entfernen)$", "", cleaned, flags=re.IGNORECASE).strip()
if not cleaned or _skip_option(cleaned):
return ""
return cleaned
def _skip_option(label: str) -> bool:
return label.strip().lower() in SKIP_OPTION_LABELS
def _first_number(text: str) -> float | int | None:
match = NUMBER_RE.search(text.replace(",", "."))
if match is None:
return None
raw = match.group(1)
value = float(raw)
return int(value) if value.is_integer() else value
def _unit(text: str) -> str | None:
match = UNIT_RE.search(text.replace(" ", ""))
if match is None:
if "kg" in text.lower():
return "kg"
return None
unit = match.group(1)
if unit in {'"', "″"}:
return "inch"
return unit
def _category_name(root: _Node, fallback: str) -> str:
titles: list[str] = []
def walk(node: _Node) -> None:
if node.tag in {"h1", "title"}:
text = _node_text(node)
if text:
titles.append(text)
for child in node.children:
if isinstance(child, _Node):
walk(child)
walk(root)
if titles:
return titles[0].split("»")[0].strip() or fallback
return "Notebooks" if fallback == "nb" else fallback

View File

@ -0,0 +1,132 @@
<!doctype html>
<html lang="de">
<head>
<title>Notebooks Preisvergleich Geizhals Deutschland</title>
</head>
<body>
<h1>Notebooks</h1>
<div data-filter-type="text_range" data-filter-id="17023">
<span id="filter-element__title-label--17023">RAM</span>
<a href="?cat=nb&xf=17023_ab+16GB&promode=true">ab 16GB</a>
<a href="?cat=nb&xf=17023_ab+32GB&promode=true">ab 32GB</a>
</div>
<form>
<label>Suchbegriffe</label>
<input name="asuch" type="text">
<label>Preisbereich (€)</label>
<input name="bpmin" type="number" placeholder="Preis ab">
<input name="bpmax" type="number" placeholder="Preis bis">
</form>
<table>
<tr>
<th>Hersteller:</th>
<td>
<a href="?cat=nb&xf=1_Apple">Apple</a>
<a href="?cat=nb&xf=1_ASUS">ASUS</a>
<a href="?cat=nb&xf=1_Lenovo">Lenovo</a>
</td>
</tr>
<tr>
<th>Display-Größe:</th>
<td>
<a href="?cat=nb&xf=12128_13">13.x"</a>
<a href="?cat=nb&xf=12128_14">14.x"</a>
<a href="?cat=nb&xf=12128_15">15.x"</a>
<a href="?cat=nb&xf=12128_16">16.x"</a>
</td>
</tr>
<tr>
<th>Display-Auflösung:</th>
<td>
<a href="?cat=nb&xf=9_1920x1080">1920x1080 (Full HD)</a>
<a href="?cat=nb&xf=9_2560x1600">2560x1600 (WQXGA)</a>
</td>
</tr>
<tr>
<th>CPU-Kerne:</th>
<td>
<a href="?cat=nb&xf=12822_8">8 (Octa-Core)</a>
<a href="?cat=nb&xf=12822_10">10</a>
<a href="?cat=nb&xf=12822_14">14-Core</a>
</td>
</tr>
<tr>
<th>CPU-Kerne ab:</th>
<td>
<a href="?cat=nb&xf=12822_8">ab 8</a>
<a href="?cat=nb&xf=12822_10">ab 10</a>
</td>
</tr>
<tr>
<th>CPU-Threads ab:</th>
<td>
<a href="?cat=nb&xf=12823_12">ab 12</a>
<a href="?cat=nb&xf=12823_16">ab 16</a>
</td>
</tr>
<tr>
<th>CPU-Turbotakt:</th>
<td>
<a href="?cat=nb&xf=9682_4500">ab 4.5GHz</a>
<a href="?cat=nb&xf=9682_5000">ab 5GHz</a>
</td>
</tr>
<tr>
<th>RAM ab:</th>
<td>
<a href="?cat=nb&xf=12_16384">ab 16GB</a>
<a href="?cat=nb&xf=12_32768">ab 32GB</a>
<a href="?cat=nb&xf=12_65536">ab 64GB</a>
</td>
</tr>
<tr>
<th>RAM-Typ:</th>
<td>
<a href="?cat=nb&xf=7520_DDR5">DDR5</a>
<a href="?cat=nb&xf=7520_LPDDR5">LPDDR5</a>
</td>
</tr>
<tr>
<th>Speichertyp:</th>
<td>
<a href="?cat=nb&xf=82_SSD">SSD</a>
<a href="?cat=nb&xf=82_M.2">M.2</a>
</td>
</tr>
<tr>
<th>SSD-Kapazität:</th>
<td>
<a href="?cat=nb&xf=2991_1000">ab 1TB</a>
<a href="?cat=nb&xf=2991_2000">ab 2TB</a>
</td>
</tr>
<tr>
<th>Grafik-Typ:</th>
<td>
<a href="?cat=nb&xf=8149_zzx">dedizierte Grafik (dGPU)</a>
</td>
</tr>
<tr>
<th>GPU - Speichergröße ab:</th>
<td>
<a href="?cat=nb&xf=7436_2048">ab 2GB</a>
<a href="?cat=nb&xf=7436_4096">ab 4GB</a>
<a href="?cat=nb&xf=7436_8192">ab 8GB</a>
</td>
</tr>
<tr>
<th>GPU - Speichergröße:</th>
<td>
<a href="?cat=nb&xf=16692_8192">8GB</a>
<a href="?cat=nb&xf=16692_12288">12GB</a>
</td>
</tr>
<tr>
<th>USB-C mit DisplayPort gesamt (inkl. Thunderbolt):</th>
<td>
<a href="?cat=nb&xf=13731_1">ab 1x</a>
</td>
</tr>
</table>
</body>
</html>

View File

@ -0,0 +1,52 @@
<!doctype html>
<html lang="de">
<body>
<a href="#productlist">6431 Produkte</a>
<nav class="pagination">
<a class="pagination__page" href="?cat=nb&amp;pg=2">Seite 2</a>
<a class="pagination__page last-page" href="?cat=nb&amp;pg=4">Seite 4</a>
</nav>
<div class="galleryview__item card">
<h3 class="galleryview__name">
<a class="galleryview__name-link" href="acme-notebook-14-a1000001.html" title="Acme Notebook 14, 32GB RAM, 1TB SSD">
Acme Notebook 14, 32GB RAM, 1TB SSD
</a>
</h3>
<span class="galleryview__mpn">ACM-14-32</span>
<dl>
<dt>Betriebssystem</dt><dd>Linux</dd>
<dt>Display</dt><dd>14"</dd>
<dt>CPU</dt><dd>Acme 8-Core</dd>
<dt>RAM</dt><dd>32GB</dd>
<dt>Speicher</dt><dd>1TB SSD</dd>
<dt>Grafik</dt><dd>Acme GPU</dd>
</dl>
<div class="galleryview__price">
<a class="galleryview__price-link" href="acme-notebook-14-a1000001.html#offerlist">
<span class="price">€ 1298,00</span>
</a>
<span class="availability" data-availability="2" aria-label="lagernd"></span>
</div>
<a class="galleryview__offercount-link">8 Angebote</a>
</div>
<div class="galleryview__item card">
<h3 class="galleryview__name">
<a class="galleryview__name-link" href="acme-notebook-13-a1000002.html" title="Acme Notebook 13, 8GB RAM">
Acme Notebook 13, 8GB RAM
</a>
</h3>
<span class="galleryview__mpn">ACM-13-8</span>
<dl>
<dt>RAM</dt><dd>8GB</dd>
<dt>Speicher</dt><dd>512GB SSD</dd>
</dl>
<div class="galleryview__price">
<a class="galleryview__price-link" href="acme-notebook-13-a1000002.html#offerlist">
<span class="price">€ 899,00</span>
</a>
<span class="availability" data-availability="1" aria-label="kurzfristig lieferbar"></span>
</div>
<a class="galleryview__offercount-link">3 Angebote</a>
</div>
</body>
</html>

View File

@ -6,6 +6,7 @@ requires-python = ">=3.13"
dependencies = [ dependencies = [
"alembic>=1.16", "alembic>=1.16",
"fastapi>=0.118", "fastapi>=0.118",
"httpx>=0.28",
"psycopg[binary]>=3.2", "psycopg[binary]>=3.2",
"pydantic>=2.11", "pydantic>=2.11",
"pydantic-settings>=2.11", "pydantic-settings>=2.11",
@ -18,6 +19,9 @@ test = [
"httpx>=0.28", "httpx>=0.28",
"pytest>=8.4", "pytest>=8.4",
] ]
live = [
"playwright>=1.48",
]
[build-system] [build-system]
requires = ["setuptools>=75"] requires = ["setuptools>=75"]

View File

@ -1,4 +1,5 @@
import json import json
import os
from datetime import datetime, timezone from datetime import datetime, timezone
from decimal import Decimal from decimal import Decimal
from pathlib import Path from pathlib import Path
@ -6,6 +7,8 @@ from uuid import UUID, uuid4
import pytest import pytest
os.environ.setdefault("GEIZHALS_USE_FIXTURES", "1")
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
from app.domain.models import ( from app.domain.models import (
AttributeDefinition, AttributeDefinition,
@ -17,7 +20,17 @@ from app.domain.models import (
) )
from app.domain.validation import parse_profile, parse_scoring_model from app.domain.validation import parse_profile, parse_scoring_model
from app.settings import get_settings
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@pytest.fixture(autouse=True)
def _geizhals_fixtures(monkeypatch):
monkeypatch.setenv("GEIZHALS_USE_FIXTURES", "1")
get_settings.cache_clear()
yield
get_settings.cache_clear()
AS_OF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc) AS_OF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)
CUTOFF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc) CUTOFF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)

View File

@ -68,6 +68,33 @@ def test_broken_mapping_is_not_used():
assert "ram_gb" in internal assert "ram_gb" in internal
def test_multiple_xf_filters_join_with_tilde():
url, _ = generate_search_url(
category_url="https://geizhals.de/?cat=nb",
criteria_items=[
{"criterion": "ram_gb", "operator": ">=", "value": 16},
{"criterion": "gpu", "operator": ">=", "value": 8},
],
mappings=[
{
"criterion_key": "ram_gb",
"external_filter_key": "12",
"source_value": "16384",
"encoding": {"url_param": "xf"},
"status": "VERIFIED",
},
{
"criterion_key": "gpu",
"external_filter_key": "7436",
"source_value": "8192",
"encoding": {"url_param": "xf"},
"status": "VERIFIED",
},
],
)
assert "xf=12_16384~7436_8192" in url or "xf=7436_8192~12_16384" in url
def test_roundtrip_verifies_xf_and_maxp(): def test_roundtrip_verifies_xf_and_maxp():
assert verify_roundtrip( assert verify_roundtrip(
category_url="https://geizhals.de/?cat=nb", category_url="https://geizhals.de/?cat=nb",

View File

@ -0,0 +1,42 @@
from pathlib import Path
from app.sources.geizhals.result_parser import parse_result_html, with_page
HTML = (Path(__file__).resolve().parents[2] / "fixtures" / "geizhals" / "nb-search-page.html").read_text(
encoding="utf-8"
)
def test_parser_reads_observed_gallery_cards() -> None:
payload = parse_result_html(HTML, page_url="https://geizhals.de/?cat=nb")
assert payload["blocked"] is False
assert payload["page_count"] == 4
assert payload["result_count"] == 6431
assert [item["source_external_id"] for item in payload["items"]] == ["1000001", "1000002"]
first = payload["items"][0]
assert first["source_url"] == "https://geizhals.de/acme-notebook-14-a1000001.html"
assert first["source_title"].startswith("Acme Notebook 14")
assert first["manufacturer"] == "Acme"
assert first["manufacturer_sku"] == "ACM-14-32"
assert first["best_price"] == "1298.00"
assert first["currency"] == "EUR"
assert first["availability"] == "in_stock"
assert first["seller_count"] == 8
assert first["raw_attributes"]["RAM"] == "32GB"
assert first["raw_attributes"]["Betriebssystem"] == "Linux"
assert first["normalized_attributes"]["ram_gb"] == 32
assert first["normalized_attributes"]["storage_gb"] == 1024
def test_challenge_html_is_not_turned_into_candidates() -> None:
payload = parse_result_html(
"<html><body>Enable JavaScript and cookies to continue</body></html>",
page_url="https://geizhals.de/?cat=nb",
)
assert payload["blocked"] is True
assert payload["items"] == []
def test_with_page_uses_geizhals_pg_parameter() -> None:
assert with_page("https://geizhals.de/?cat=nb&xf=17023_ab+16GB", 1) == "https://geizhals.de/?cat=nb&xf=17023_ab+16GB"
assert "pg=3" in with_page("https://geizhals.de/?cat=nb&xf=17023_ab+16GB", 3)

View File

@ -0,0 +1,76 @@
from pathlib import Path
from app.discovery.urlgen import generate_search_url
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
HTML = (Path(__file__).resolve().parents[2] / "fixtures" / "geizhals" / "nb-category.html").read_text(
encoding="utf-8"
)
def test_parser_reads_href_filters_and_price_inputs() -> None:
payload = parse_taxonomy_html(
HTML,
category_ref="nb",
source_url="https://geizhals.de/?cat=nb",
)
keys = {item["external_filter_key"] for item in payload["filters"]}
assert {"12", "17023", "price", "12128", "7436", "82", "asuch"} <= keys
assert len(payload["filters"]) >= 12
live_ram = next(item for item in payload["filters"] if item["external_filter_key"] == "17023")
assert live_ram["label"] == "RAM"
ram = next(item for item in payload["filters"] if item["external_filter_key"] == "12")
assert ram["label"] == "RAM ab"
assert ram["filter_type"] == "NUMERIC_THRESHOLD"
assert any(option["source_value"] == "ab 16GB" for option in ram["options"])
assert ram["encoding"]["options"]["ab 16GB"] == "16384"
price = next(item for item in payload["filters"] if item["external_filter_key"] == "price")
assert price["filter_type"] == "NUMERIC_RANGE"
assert price["encoding"]["min_parameter"] == "bpmin"
assert price["encoding"]["max_parameter"] == "bpmax"
def test_learned_ram_and_price_encode_observed_geizhals_query() -> None:
payload = parse_taxonomy_html(
HTML,
category_ref="nb",
source_url="https://geizhals.de/?cat=nb",
)
ram = next(item for item in payload["filters"] if item["external_filter_key"] == "12")
price = next(item for item in payload["filters"] if item["external_filter_key"] == "price")
url, internal = generate_search_url(
category_url="https://geizhals.de/?cat=nb",
criteria_items=[
{"criterion": "geizhals:nb:12", "operator": ">=", "value": 16},
{"criterion": "geizhals:nb:price", "operator": "<=", "value": 1700},
],
mappings=[
{
"criterion_key": "geizhals:nb:12",
"external_filter_key": "12",
"source_value": "16384",
"encoding": ram["encoding"],
"status": "VERIFIED",
},
{
"criterion_key": "geizhals:nb:price",
"external_filter_key": "price",
"source_value": None,
"encoding": price["encoding"],
"status": "VERIFIED",
},
],
)
assert internal == []
assert "xf=12_16384" in url
assert "bpmax=1700" in url
def test_challenge_html_is_not_treated_as_taxonomy() -> None:
payload = parse_taxonomy_html(
"<html><body>Enable JavaScript and cookies to continue</body></html>",
category_ref="nb",
source_url="https://geizhals.de/?cat=nb",
)
assert payload["blocked"] is True
assert payload["filters"] == []

View File

@ -12,3 +12,9 @@ def test_database_url_uses_psycopg_and_explicit_fields() -> None:
assert settings.database_url == ( assert settings.database_url == (
"postgresql+psycopg://buyingagent:secret@postgres:5432/buyingagent_test" "postgresql+psycopg://buyingagent:secret@postgres:5432/buyingagent_test"
) )
def test_geizhals_live_defaults_are_off_for_fixtures_flag() -> None:
settings = Settings(geizhals_use_fixtures=True, geizhals_playwright=False)
assert settings.geizhals_use_fixtures is True
assert settings.geizhals_playwright is False

View File

@ -54,6 +54,7 @@ source = { editable = "." }
dependencies = [ dependencies = [
{ name = "alembic" }, { name = "alembic" },
{ name = "fastapi" }, { name = "fastapi" },
{ name = "httpx" },
{ name = "psycopg", extra = ["binary"] }, { name = "psycopg", extra = ["binary"] },
{ name = "pydantic" }, { name = "pydantic" },
{ name = "pydantic-settings" }, { name = "pydantic-settings" },
@ -62,6 +63,9 @@ dependencies = [
] ]
[package.optional-dependencies] [package.optional-dependencies]
live = [
{ name = "playwright" },
]
test = [ test = [
{ name = "httpx" }, { name = "httpx" },
{ name = "pytest" }, { name = "pytest" },
@ -71,7 +75,9 @@ test = [
requires-dist = [ requires-dist = [
{ name = "alembic", specifier = ">=1.16" }, { name = "alembic", specifier = ">=1.16" },
{ name = "fastapi", specifier = ">=0.118" }, { name = "fastapi", specifier = ">=0.118" },
{ name = "httpx", specifier = ">=0.28" },
{ name = "httpx", marker = "extra == 'test'", specifier = ">=0.28" }, { name = "httpx", marker = "extra == 'test'", specifier = ">=0.28" },
{ name = "playwright", marker = "extra == 'live'", specifier = ">=1.48" },
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2" },
{ name = "pydantic", specifier = ">=2.11" }, { name = "pydantic", specifier = ">=2.11" },
{ name = "pydantic-settings", specifier = ">=2.11" }, { name = "pydantic-settings", specifier = ">=2.11" },
@ -79,7 +85,7 @@ requires-dist = [
{ name = "sqlalchemy", specifier = ">=2.0" }, { name = "sqlalchemy", specifier = ">=2.0" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.37" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.37" },
] ]
provides-extras = ["test"] provides-extras = ["test", "live"]
[[package]] [[package]]
name = "certifi" name = "certifi"
@ -125,6 +131,63 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" }, { url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" },
] ]
[[package]]
name = "greenlet"
version = "3.5.6"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/a1/e720a38852366c589e1a46cf570b886507ad2cf591050c203365638baab0/greenlet-3.5.6-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:f96f0e30b5a95c7631b12bfe214cbc90ec8fe8cfa36920596c10514a65743519", size = 294627, upload-time = "2026-09-14T14:24:40.102Z" },
{ url = "https://files.pythonhosted.org/packages/eb/c3/58187858df41354a11e6a55b421e7af9059798abdab3a384cc51b8567c38/greenlet-3.5.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c75116c9de79949de23006e2d9b35ee82874c594fcf5c0311b439acaa14b8441", size = 614356, upload-time = "2026-09-14T15:12:03.399Z" },
{ url = "https://files.pythonhosted.org/packages/ce/b9/3a7e67d5f05c9760b1ad411fa52264bd69cc08e22a2ebfb4018b90628ced/greenlet-3.5.6-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cad5782f93f7f738b62c6527b6f32a60694d924029f299a8b524758cfa53d815", size = 626756, upload-time = "2026-09-14T15:20:44.269Z" },
{ url = "https://files.pythonhosted.org/packages/c6/7c/40400455f5b5a65bb83e94fde66d1be9e5ec518638113f8083ace746c309/greenlet-3.5.6-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a93ee7c6e8fd0f8a83525a51bd777be57ee17787e91d805bd8d6faf9dcada18e", size = 632632, upload-time = "2026-09-14T15:25:07.813Z" },
{ url = "https://files.pythonhosted.org/packages/85/cb/ab0c123c514ed4e94c0dc9ee2e86362633e6b998cfc05de7fc9ac2eb9690/greenlet-3.5.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f98e8215e172f567ce80eeaed9107fb4d32b6c44f26983d9b8334658136a205a", size = 623779, upload-time = "2026-09-14T14:36:01.104Z" },
{ url = "https://files.pythonhosted.org/packages/f9/67/1f35cff30a6c51c3f23b63d4afcc7313ab4f97490ba3676fa78178984b27/greenlet-3.5.6-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7f731ebac68ea06d628658295cb2d217b10186329fcf9a3b6a149045059bf92e", size = 434933, upload-time = "2026-09-14T15:28:38.858Z" },
{ url = "https://files.pythonhosted.org/packages/a5/26/fda8a5a06e7073333ccb038133c5893b9e0c4fe29d5992a17e83c241bc6e/greenlet-3.5.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df19e2d0b1620039af5102563fbd96e8938c7f5c3f5828528d641d9fc585525e", size = 1584930, upload-time = "2026-09-14T15:10:08.234Z" },
{ url = "https://files.pythonhosted.org/packages/2f/37/50f8813163148d6234e08b23dcad6a9e37f01d148c8ec976e4c44ea2d918/greenlet-3.5.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06c0e933290fba8ffe53ead4ae1b8044b0e9754b75cebf381aa2bc3e50d82fac", size = 1647590, upload-time = "2026-09-14T14:35:51.173Z" },
{ url = "https://files.pythonhosted.org/packages/86/da/b7669b09586365654083a62bd0724cf06cb74bd5085a15cdd161271f992f/greenlet-3.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:5b602b4201b965a8354d74e232364a66ff243dd142e350d035f46169bb36e13d", size = 324086, upload-time = "2026-09-14T14:23:48.428Z" },
{ url = "https://files.pythonhosted.org/packages/e5/5d/c9663cfe84a2a9e0aa96f066f5b0594c227ea4c647511e087e2e11d4ac0a/greenlet-3.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:876077e7ebb8c84ed068e2b23d4c62ebb010d60df84b9591af1be2f39010ffb2", size = 308211, upload-time = "2026-09-14T14:28:01.634Z" },
{ url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" },
{ url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" },
{ url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" },
{ url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" },
{ url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" },
{ url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" },
{ url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" },
{ url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" },
{ url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" },
{ url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" },
{ url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" },
{ url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" },
{ url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" },
{ url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" },
{ url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" },
{ url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" },
{ url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" },
{ url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" },
{ url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" },
{ url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" },
{ url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" },
{ url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" },
{ url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" },
{ url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" },
{ url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" },
{ url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" },
{ url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" },
{ url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" },
{ url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" },
{ url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" },
{ url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" },
{ url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" },
{ url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" },
{ url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" },
{ url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" },
{ url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" },
{ url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" },
{ url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" },
{ url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" },
]
[[package]] [[package]]
name = "h11" name = "h11"
version = "0.16.0" version = "0.16.0"
@ -340,6 +403,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
] ]
[[package]]
name = "playwright"
version = "1.63.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "greenlet" },
{ name = "pyee" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/19/dd/fbb3d34228ad753bc14464d5f2252585b73367539775a88d0a3949cf5a45/playwright-1.63.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:84c540759e8e7f01e690e197e04060f48899d37cea322299f255843273d3385d", size = 44278407, upload-time = "2026-09-15T16:49:06.045Z" },
{ url = "https://files.pythonhosted.org/packages/f5/9a/948b930b1a8c4ee869e5a139a2b7747caa06aab56a3f09a2f0abdcbda221/playwright-1.63.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fd1aa00631d44d55e56e0975bf3f3f285fac4a9fd2813183f0c12a488f1a1b24", size = 42944679, upload-time = "2026-09-15T16:49:10.039Z" },
{ url = "https://files.pythonhosted.org/packages/94/11/dc5c13fa1602371603acd461be47529c1b3513815d3a0dc98f642c291a10/playwright-1.63.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:c89fc4736502a1f0fac2c8ca5d10c0cbc1c669f1f4774a2d8507a43140e4d53f", size = 44278410, upload-time = "2026-09-15T16:49:13.861Z" },
{ url = "https://files.pythonhosted.org/packages/27/9c/103a5037789062bdab27c7dca53f3ca6b075b572ab2cd96eec825b3aec4e/playwright-1.63.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ad21bc07516b187965a7521c5cf0df0bd657b17482eaad74335272d35a2b07de", size = 48217159, upload-time = "2026-09-15T16:49:17.404Z" },
{ url = "https://files.pythonhosted.org/packages/f3/82/3d85505284c5a210f2da6c07b8f757524e79d1fba9cfdafe1eafb766ae59/playwright-1.63.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:354e15b29503565fc598b89f16fbe070459343bef9d7498a93e304864000c6a7", size = 47902708, upload-time = "2026-09-15T16:49:21.055Z" },
{ url = "https://files.pythonhosted.org/packages/69/8d/f74ff6b52751859f4b69caf66aa7d4a3c8d6c1f0c7dc9920da103612ee39/playwright-1.63.0-py3-none-win32.whl", hash = "sha256:660c00c62639e31b16700ba5456b351ddba55bb766b7ce8261223aa34928e482", size = 38606075, upload-time = "2026-09-15T16:49:29.546Z" },
{ url = "https://files.pythonhosted.org/packages/76/eb/d6b8d92658038e260dbc7dd69fb3fdbe245aac283953de8b50b02bfe5f61/playwright-1.63.0-py3-none-win_amd64.whl", hash = "sha256:2f9a707a6c6c91157ed77bff2b8caeb04b3c8d46e70d585fc134298cbe4b5cc6", size = 38606082, upload-time = "2026-09-15T16:49:32.838Z" },
{ url = "https://files.pythonhosted.org/packages/be/75/2432b3e3c7c62103b72d4c4cc8b16a56383ada372bbb0d1278591e988f71/playwright-1.63.0-py3-none-win_arm64.whl", hash = "sha256:1e4a3a838ce22fb68ad17193fcd142a19610d9d70fb9966d2239f5dddc0cc05b", size = 34523554, upload-time = "2026-09-15T16:49:36.455Z" },
]
[[package]] [[package]]
name = "pluggy" name = "pluggy"
version = "1.6.0" version = "1.6.0"
@ -491,6 +573,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" },
] ]
[[package]]
name = "pyee"
version = "13.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
]
[[package]] [[package]]
name = "pygments" name = "pygments"
version = "2.21.0" version = "2.21.0"

View File

@ -33,6 +33,8 @@ services:
ENVIRONMENT: "${ENVIRONMENT:-development}" ENVIRONMENT: "${ENVIRONMENT:-development}"
ENGINE_BUILD: "${ENGINE_BUILD:-dev}" ENGINE_BUILD: "${ENGINE_BUILD:-dev}"
CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:5173,http://127.0.0.1:5173,http://192.168.2.49:5174}" CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:5173,http://127.0.0.1:5173,http://192.168.2.49:5174}"
GEIZHALS_USE_FIXTURES: "${GEIZHALS_USE_FIXTURES:-0}"
GEIZHALS_PLAYWRIGHT: "${GEIZHALS_PLAYWRIGHT:-1}"
ports: ports:
- "${BUYING_AGENT_API_PORT:-8095}:8000" - "${BUYING_AGENT_API_PORT:-8095}:8000"
depends_on: depends_on:
@ -84,6 +86,7 @@ services:
ENVIRONMENT: test ENVIRONMENT: test
ENGINE_BUILD: test ENGINE_BUILD: test
CLOCK_NOW: "2026-10-03T12:00:00Z" CLOCK_NOW: "2026-10-03T12:00:00Z"
GEIZHALS_USE_FIXTURES: "1"
depends_on: depends_on:
postgres: postgres:
condition: service_healthy condition: service_healthy

View File

@ -34,6 +34,7 @@ services:
ENVIRONMENT: test ENVIRONMENT: test
ENGINE_BUILD: test ENGINE_BUILD: test
CLOCK_NOW: "2026-10-03T12:00:00Z" CLOCK_NOW: "2026-10-03T12:00:00Z"
GEIZHALS_USE_FIXTURES: "1"
depends_on: depends_on:
postgres: postgres:
condition: service_healthy condition: service_healthy

View File

@ -36,6 +36,8 @@ services:
ENVIRONMENT: "${ENVIRONMENT:-production}" ENVIRONMENT: "${ENVIRONMENT:-production}"
ENGINE_BUILD: "${ENGINE_BUILD:-dev}" ENGINE_BUILD: "${ENGINE_BUILD:-dev}"
CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:4173,http://127.0.0.1:4173}" CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:4173,http://127.0.0.1:4173}"
GEIZHALS_USE_FIXTURES: "${GEIZHALS_USE_FIXTURES:-0}"
GEIZHALS_PLAYWRIGHT: "${GEIZHALS_PLAYWRIGHT:-1}"
ports: ports:
- "${BUYING_AGENT_API_BIND:-127.0.0.1}:${BUYING_AGENT_API_PORT:-8000}:8000" - "${BUYING_AGENT_API_BIND:-127.0.0.1}:${BUYING_AGENT_API_PORT:-8000}:8000"
depends_on: depends_on:

View File

@ -8,6 +8,9 @@ server {
proxy_pass http://api:8000; proxy_pass http://api:8000;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 10s;
proxy_read_timeout 120s;
proxy_send_timeout 120s;
} }
location /health { location /health {

View File

@ -7,7 +7,7 @@ async function request<T>(path: string, init?: RequestInit): Promise<T> {
}); });
const body = await response.json().catch(() => ({})); const body = await response.json().catch(() => ({}));
if (!response.ok) { if (!response.ok) {
throw new Error(body.message || `HTTP ${response.status}`); throw new Error(body?.error?.message || body?.message || `HTTP ${response.status}`);
} }
return body as T; return body as T;
} }
@ -101,7 +101,12 @@ export type AvailableCriterion = {
supports_discovery: boolean; supports_discovery: boolean;
scoring_attribute_key?: string | null; scoring_attribute_key?: string | null;
}; };
export type AvailableCriteria = { product_category_id: string; source: string; criteria: AvailableCriterion[] }; export type AvailableCriteria = {
product_category_id: string;
source: string;
taxonomy_status?: string | null;
criteria: AvailableCriterion[];
};
export type DiscoverySearch = { export type DiscoverySearch = {
id: string; id: string;
status: string; status: string;

View File

@ -33,9 +33,10 @@ export function AdminPage() {
<p>Source: {source || "–"}</p> <p>Source: {source || "–"}</p>
<p>Kategorie: {category || "–"}</p> <p>Kategorie: {category || "–"}</p>
<p>Status: {selected?.taxonomy_status || "NOT_LEARNED"}</p> <p>Status: {selected?.taxonomy_status || "NOT_LEARNED"}</p>
<p>Gelernte Filter: {filters.data?.filters.length ?? 0}</p>
{discover.isError ? <p className="danger">{(discover.error as Error).message}</p> : null} {discover.isError ? <p className="danger">{(discover.error as Error).message}</p> : null}
<button className="button" disabled={!source || !category || discover.isPending} onClick={() => discover.mutate()}> <button className="button" disabled={!source || !category || discover.isPending} onClick={() => discover.mutate()}>
Taxonomie laden {discover.isPending ? "Lernt Taxonomie …" : "Taxonomie neu lernen"}
</button> </button>
</div> </div>
{!category ? ( {!category ? (

View File

@ -18,6 +18,7 @@ export function WizardPage() {
const [sourceOverride, setSourceOverride] = useState<string | null>(null); const [sourceOverride, setSourceOverride] = useState<string | null>(null);
const [selected, setSelected] = useState<string[]>([]); const [selected, setSelected] = useState<string[]>([]);
const [draft, setDraft] = useState<Draft>({}); const [draft, setDraft] = useState<Draft>({});
const [criterionQuery, setCriterionQuery] = useState("");
const sources = useQuery({ const sources = useQuery({
queryKey: ["category-sources", categoryId], queryKey: ["category-sources", categoryId],
@ -63,7 +64,12 @@ export function WizardPage() {
const sourceRow = sourceItems.find((item) => item.source === source) || sourceItems[0]; const sourceRow = sourceItems.find((item) => item.source === source) || sourceItems[0];
const needsPicker = sourceItems.filter((item) => item.status === "ACTIVE").length > 1; const needsPicker = sourceItems.filter((item) => item.status === "ACTIVE").length > 1;
const visible = useMemo(() => criteria.data?.criteria || [], [criteria.data]); const visible = useMemo(() => {
const items = criteria.data?.criteria || [];
const query = criterionQuery.trim().toLowerCase();
if (!query) return items;
return items.filter((item) => item.label.toLowerCase().includes(query));
}, [criteria.data, criterionQuery]);
if (categories.isLoading) return <LoadingState />; if (categories.isLoading) return <LoadingState />;
if (categories.isError) return <ErrorState message={(categories.error as Error).message} />; if (categories.isError) return <ErrorState message={(categories.error as Error).message} />;
@ -84,6 +90,7 @@ export function WizardPage() {
setCategoryId(event.target.value); setCategoryId(event.target.value);
setSourceOverride(null); setSourceOverride(null);
setSelected([]); setSelected([]);
setCriterionQuery("");
}} }}
> >
<option value="">Bitte wählen</option> <option value="">Bitte wählen</option>
@ -139,10 +146,21 @@ export function WizardPage() {
<p className="muted">Quelle: {source}</p> <p className="muted">Quelle: {source}</p>
) : null} ) : null}
{(criteria.data?.taxonomy_status || sourceRow?.taxonomy_status) === "DEGRADED" ? (
<p className="muted">Die Taxonomie stammt aus einer aufgezeichneten Kategorieseite. Live-Lernen war blockiert.</p>
) : null}
{criteria.isError ? <ErrorState message={(criteria.error as Error).message} onRetry={() => criteria.refetch()} /> : null} {criteria.isError ? <ErrorState message={(criteria.error as Error).message} onRetry={() => criteria.refetch()} /> : null}
{criteria.isLoading ? <LoadingState /> : null} {criteria.isLoading ? <LoadingState /> : null}
{visible.length > 0 ? ( {(criteria.data?.criteria || []).length > 0 ? (
<div className="card grid"> <div className="card grid">
<p>
{(criteria.data?.criteria || []).length} Kriterien aus der Quelle. Wähle die relevanten aus und setze
Grenzwerte.
</p>
<label>
Kriterien suchen
<input value={criterionQuery} onChange={(event) => setCriterionQuery(event.target.value)} />
</label>
{visible.map((item) => { {visible.map((item) => {
const on = selected.includes(item.criterion_definition_id); const on = selected.includes(item.criterion_definition_id);
return ( return (