Lerne Taxonomie und Trefferliste live von Geizhals, damit Wizard und Kandidaten echte Quelldaten nutzen.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
d784248e28
commit
46857715df
|
|
@ -18,11 +18,13 @@ RUN uv sync --frozen --no-dev --no-install-project
|
|||
FROM base AS runtime
|
||||
|
||||
COPY . .
|
||||
RUN uv sync --frozen --no-dev \
|
||||
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
|
||||
RUN uv sync --frozen --extra live --no-dev \
|
||||
&& playwright install --with-deps chromium \
|
||||
&& sed -i 's/\r$//' /app/scripts/startup.sh \
|
||||
&& chmod +x /app/scripts/startup.sh \
|
||||
&& useradd --create-home --uid 1000 app \
|
||||
&& chown -R app:app /app
|
||||
&& chown -R app:app /app /ms-playwright
|
||||
|
||||
USER app
|
||||
EXPOSE 8000
|
||||
|
|
|
|||
|
|
@ -27,11 +27,17 @@ def generate_search_url(
|
|||
base = urlsplit(category_url)
|
||||
query = dict(parse_qsl(base.query, keep_blank_values=True))
|
||||
extra: list[tuple[str, str]] = []
|
||||
xf_values: list[str] = []
|
||||
for name, value in fragments:
|
||||
if name in query and name != "xf":
|
||||
if name == "xf":
|
||||
xf_values.append(value)
|
||||
continue
|
||||
if name in query:
|
||||
query[name] = value
|
||||
else:
|
||||
extra.append((name, value))
|
||||
if xf_values:
|
||||
extra.append(("xf", "~".join(xf_values)))
|
||||
encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote)
|
||||
url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment))
|
||||
return url, internal_only
|
||||
|
|
@ -40,11 +46,15 @@ def generate_search_url(
|
|||
def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
encoding = mapping.get("encoding") or {}
|
||||
param = str(encoding.get("url_param") or "xf")
|
||||
if encoding.get("encoding_type") == "RANGE_PARAMETERS" or param in {"bpmin", "bpmax"}:
|
||||
return _encode_range(encoding, criterion)
|
||||
if encoding.get("encoding_type") == "RANGE_TOKEN":
|
||||
return _encode_range_token(mapping, criterion)
|
||||
if param == "xf":
|
||||
filter_key = str(mapping["external_filter_key"])
|
||||
source_value = str(mapping.get("source_value") or _threshold_value(mapping, criterion))
|
||||
source_value = _xf_value(mapping, criterion)
|
||||
return [("xf", f"{filter_key}_{_plus(source_value)}")]
|
||||
if param in {"minp", "maxp", "prmin", "prmax"}:
|
||||
if param in {"minp", "maxp", "prmin", "prmax", "asuch", "fs"}:
|
||||
return [(param, str(criterion.get("value")))]
|
||||
template = encoding.get("value_template")
|
||||
if template:
|
||||
|
|
@ -52,6 +62,63 @@ def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[
|
|||
raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}")
|
||||
|
||||
|
||||
def _encode_range(encoding: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
operator = str(criterion.get("operator") or "<=")
|
||||
value = criterion.get("value")
|
||||
min_param = str(encoding.get("min_parameter") or encoding.get("url_param") or "bpmin")
|
||||
max_param = str(encoding.get("max_parameter") or "bpmax")
|
||||
if operator == "between":
|
||||
if isinstance(value, dict):
|
||||
items: list[tuple[str, str]] = []
|
||||
if value.get("min") is not None:
|
||||
items.append((min_param, str(value["min"])))
|
||||
if value.get("max") is not None:
|
||||
items.append((max_param, str(value["max"])))
|
||||
return items
|
||||
if isinstance(value, list) and len(value) == 2:
|
||||
return [(min_param, str(value[0])), (max_param, str(value[1]))]
|
||||
if operator in {">=", ">"}:
|
||||
return [(min_param, str(value))]
|
||||
return [(max_param, str(value))]
|
||||
|
||||
|
||||
def _encode_range_token(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
filter_key = str(mapping["external_filter_key"])
|
||||
looked_up = _lookup_option(mapping, criterion.get("value"))
|
||||
if looked_up is not None:
|
||||
return [("xf", f"{filter_key}_{_plus(looked_up)}")]
|
||||
operator = str(criterion.get("operator") or "<=")
|
||||
value = criterion.get("value")
|
||||
if operator == "between":
|
||||
if isinstance(value, dict):
|
||||
raw = f"{value.get('min') or ''}*{value.get('max') or ''}"
|
||||
elif isinstance(value, list) and len(value) == 2:
|
||||
raw = f"{value[0]}*{value[1]}"
|
||||
else:
|
||||
raw = f"*{value}"
|
||||
elif operator in {">=", ">"}:
|
||||
raw = f"{value}*"
|
||||
else:
|
||||
raw = f"*{value}"
|
||||
return [("xf", f"{filter_key}_{raw}")]
|
||||
|
||||
|
||||
def _lookup_option(mapping: dict[str, Any], value: Any) -> str | None:
|
||||
options = (mapping.get("encoding") or {}).get("options") or {}
|
||||
if value in options:
|
||||
return str(options[value])
|
||||
if str(value) in options:
|
||||
return str(options[str(value)])
|
||||
return None
|
||||
|
||||
|
||||
def _xf_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
||||
looked_up = _lookup_option(mapping, criterion.get("value"))
|
||||
if looked_up is not None:
|
||||
return looked_up
|
||||
return str(mapping.get("source_value") or _threshold_value(mapping, criterion))
|
||||
|
||||
|
||||
def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
||||
option = mapping.get("source_value")
|
||||
if option:
|
||||
|
|
@ -72,8 +139,12 @@ def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]:
|
|||
parts = urlsplit(url)
|
||||
decoded: list[tuple[str, str]] = []
|
||||
for name, value in parse_qsl(parts.query, keep_blank_values=True):
|
||||
if name == "xf" and "_" in value:
|
||||
key, raw = value.split("_", 1)
|
||||
if name != "xf":
|
||||
continue
|
||||
for part in value.replace("|", "~").split("~"):
|
||||
if "_" not in part:
|
||||
continue
|
||||
key, raw = part.split("_", 1)
|
||||
decoded.append((key, raw.replace("+", " ")))
|
||||
return decoded
|
||||
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ class CatalogService:
|
|||
SourceCategoryRecord.product_category_id == product_category_id,
|
||||
)
|
||||
).first()
|
||||
if source_category is not None and source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
|
||||
if source_category is not None and self._should_learn(source_category, source_id):
|
||||
from app.services.discovery_service import TaxonomyService
|
||||
|
||||
TaxonomyService(self.session).discover(source_id, source_category.external_category_key)
|
||||
|
|
@ -172,7 +172,33 @@ class CatalogService:
|
|||
if definition.id in seen:
|
||||
continue
|
||||
criteria.append(self._criterion_out(definition, None, None, [], None))
|
||||
return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria})
|
||||
return jsonable(
|
||||
{
|
||||
"product_category_id": product_category_id,
|
||||
"source": source_id,
|
||||
"taxonomy_status": None if source_category is None else source_category.taxonomy_status,
|
||||
"criteria": criteria,
|
||||
}
|
||||
)
|
||||
|
||||
def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool:
|
||||
if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
|
||||
return True
|
||||
if source_category.taxonomy_status != TaxonomyStatus.READY.value:
|
||||
return False
|
||||
from app.settings import get_settings
|
||||
|
||||
if get_settings().geizhals_use_fixtures or source_id != "geizhals":
|
||||
return False
|
||||
keys = {
|
||||
row.external_filter_key
|
||||
for row in self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord).where(
|
||||
SourceFilterDefinitionRecord.source_category_id == source_category.id
|
||||
)
|
||||
)
|
||||
}
|
||||
return keys <= {"17023", "price"}
|
||||
|
||||
def list_templates(self) -> list[dict[str, Any]]:
|
||||
rows, _total = self.catalog.list_profiles(200, 0)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from app.discovery.diff import diff_run, next_lifecycle
|
|||
from app.discovery.enums import (
|
||||
CandidateStatus,
|
||||
DiffStatus,
|
||||
FilterType,
|
||||
LifecycleStatus,
|
||||
MappingStatus,
|
||||
MatchStatus,
|
||||
|
|
@ -99,7 +100,20 @@ class TaxonomyService:
|
|||
if catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
adapter = self.adapter or discovery_adapter(source_id)
|
||||
try:
|
||||
payload = adapter.discover_taxonomy(category_key)
|
||||
except AppError as exc:
|
||||
category = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.external_category_key == category_key,
|
||||
)
|
||||
).first()
|
||||
if category is not None and exc.code == "SOURCE_BLOCKED":
|
||||
category.taxonomy_status = TaxonomyStatus.BLOCKED.value
|
||||
category.last_taxonomy_scan_at = now_utc()
|
||||
self.session.flush()
|
||||
raise
|
||||
now = now_utc()
|
||||
category = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
|
|
@ -148,7 +162,7 @@ class TaxonomyService:
|
|||
label=str(raw.get("label") or raw["external_filter_key"]),
|
||||
filter_type=filter_type,
|
||||
canonical_unit=raw.get("canonical_unit"),
|
||||
status=MappingStatus.VERIFIED.value,
|
||||
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
||||
)
|
||||
self.session.add(filt)
|
||||
self.session.flush()
|
||||
|
|
@ -161,6 +175,11 @@ class TaxonomyService:
|
|||
self._sync_options(filt, raw.get("options") or [])
|
||||
if raw.get("range"):
|
||||
self._sync_range(filt, raw["range"])
|
||||
self._sync_learned_mapping(category, filt, raw)
|
||||
requested = str(payload.get("taxonomy_status") or "")
|
||||
if requested in {item.value for item in TaxonomyStatus}:
|
||||
category.taxonomy_status = requested
|
||||
else:
|
||||
category.taxonomy_status = (
|
||||
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
|
||||
)
|
||||
|
|
@ -275,6 +294,89 @@ class TaxonomyService:
|
|||
if removed:
|
||||
self._break_mappings(filt.source_category_id, filt.external_filter_key)
|
||||
|
||||
def _sync_learned_mapping(
|
||||
self,
|
||||
category: SourceCategoryRecord,
|
||||
filt: SourceFilterDefinitionRecord,
|
||||
raw: dict[str, Any],
|
||||
) -> None:
|
||||
encoding = raw.get("encoding") or {"url_param": "xf"}
|
||||
definition = upsert_definition(
|
||||
self.session,
|
||||
{
|
||||
"key": f"{category.source_id}:{category.external_category_key}:{filt.external_filter_key}",
|
||||
"origin": "SOURCE",
|
||||
"source": category.source_id,
|
||||
"product_category_id": category.product_category_id,
|
||||
"label": filt.label,
|
||||
"value_type": "number"
|
||||
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
|
||||
else "string",
|
||||
"unit": filt.canonical_unit,
|
||||
"category": category.product_category_id,
|
||||
},
|
||||
)
|
||||
mapping = self.session.scalars(
|
||||
select(CriterionMappingRecord).where(
|
||||
CriterionMappingRecord.source_category_id == category.id,
|
||||
CriterionMappingRecord.external_filter_key == filt.external_filter_key,
|
||||
)
|
||||
).first()
|
||||
options = raw.get("options") or []
|
||||
source_value = None
|
||||
if options:
|
||||
source_value = str(options[0].get("query_value") or options[0].get("source_value"))
|
||||
now = now_utc()
|
||||
if mapping is None:
|
||||
mapping = CriterionMappingRecord(
|
||||
id=uuid4(),
|
||||
criterion_definition_id=definition.id,
|
||||
criterion_key=definition.key,
|
||||
source_id=category.source_id,
|
||||
source_category_id=category.id,
|
||||
external_filter_key=filt.external_filter_key,
|
||||
source_value=source_value,
|
||||
encoding=encoding,
|
||||
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
||||
definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
self.session.add(mapping)
|
||||
self.session.flush()
|
||||
else:
|
||||
mapping.criterion_definition_id = definition.id
|
||||
mapping.criterion_key = definition.key
|
||||
mapping.source_value = source_value
|
||||
mapping.encoding = encoding
|
||||
mapping.definition_hash = sha256_text(canonical_json({"encoding": encoding, "source_value": source_value}))
|
||||
mapping.updated_at = now
|
||||
probe_value = source_value or 0
|
||||
if options and options[0].get("normalized_value") is not None:
|
||||
probe_value = options[0]["normalized_value"]
|
||||
operator = (options[0].get("operator") if options else None) or "<="
|
||||
if filt.filter_type == FilterType.NUMERIC_RANGE.value:
|
||||
operator = ">="
|
||||
probe_value = 1000
|
||||
verified = verify_roundtrip(
|
||||
category_url=category.source_url or f"https://geizhals.de/?cat={category.external_category_key}",
|
||||
criterion={
|
||||
"criterion": definition.key,
|
||||
"operator": operator,
|
||||
"value": probe_value,
|
||||
},
|
||||
mapping={
|
||||
"criterion_key": definition.key,
|
||||
"external_filter_key": filt.external_filter_key,
|
||||
"source_value": source_value,
|
||||
"encoding": encoding,
|
||||
},
|
||||
)
|
||||
if mapping.status != MappingStatus.DISABLED.value and verified:
|
||||
mapping.status = MappingStatus.VERIFIED.value
|
||||
elif mapping.status == MappingStatus.BROKEN.value:
|
||||
mapping.status = MappingStatus.LEARNED_UNVERIFIED.value
|
||||
|
||||
def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None:
|
||||
row = self.session.get(SourceFilterRangeRecord, filt.id)
|
||||
if row is None:
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ class Settings(BaseSettings):
|
|||
engine_build: str = "dev"
|
||||
clock_now: datetime | None = None
|
||||
cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173"
|
||||
geizhals_use_fixtures: bool = False
|
||||
geizhals_playwright: bool = True
|
||||
|
||||
@property
|
||||
def cors_origins(self) -> list[str]:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,10 @@ from urllib.parse import parse_qs, urlparse
|
|||
|
||||
from app.discovery.enums import FilterType
|
||||
from app.errors import AppError
|
||||
from app.settings import get_settings
|
||||
from app.sources.geizhals.live_page import load_category_page, load_result_page
|
||||
from app.sources.geizhals.result_parser import parse_result_html, with_page
|
||||
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
||||
|
||||
|
|
@ -14,12 +18,31 @@ FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
|||
class GeizhalsAdapter:
|
||||
source_key = "geizhals"
|
||||
|
||||
def __init__(self, fixture_dir: Path | None = None, live: bool = False) -> None:
|
||||
def __init__(self, fixture_dir: Path | None = None, live: bool | None = None) -> None:
|
||||
self.fixture_dir = fixture_dir or FIXTURE_DIR
|
||||
self.live = live
|
||||
settings = get_settings()
|
||||
self.live = (not settings.geizhals_use_fixtures) if live is None else live
|
||||
|
||||
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]:
|
||||
payload = self._load(f"{category_ref}-taxonomy.json")
|
||||
if not self.live:
|
||||
return self._load(f"{category_ref}-taxonomy.json")
|
||||
url = f"https://geizhals.de/?cat={category_ref}&promode=true"
|
||||
html, mode, status = load_category_page(
|
||||
url,
|
||||
category_ref=category_ref,
|
||||
playwright=get_settings().geizhals_playwright,
|
||||
)
|
||||
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=url)
|
||||
if payload.get("blocked") or not payload.get("filters"):
|
||||
raise AppError(
|
||||
422,
|
||||
"TAXONOMY_EMPTY",
|
||||
"Geizhals-Taxonomie konnte nicht aus der Kategorieseite gelesen werden",
|
||||
{"url": url, "http_status": status, "mode": mode},
|
||||
)
|
||||
payload["discovery_mode"] = mode
|
||||
payload["http_status"] = status
|
||||
payload["taxonomy_status"] = "DEGRADED" if mode == "recorded_html" else "READY"
|
||||
return payload
|
||||
|
||||
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
|
|
@ -34,13 +57,32 @@ class GeizhalsAdapter:
|
|||
return {"url": url, "internal_only": internal_only}
|
||||
|
||||
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]:
|
||||
if self.live:
|
||||
raise AppError(422, "LIVE_FETCH_DISABLED", "live Geizhals fetch is not enabled in this process")
|
||||
page = int(pagination.get("page") or 1)
|
||||
if not self.live:
|
||||
category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0]
|
||||
return self._load(f"{category}-search-page{page}.json")
|
||||
url = with_page(str(query["url"]), page)
|
||||
html, mode, status, blocked = load_result_page(url, playwright=get_settings().geizhals_playwright)
|
||||
if blocked:
|
||||
return {
|
||||
"page": page,
|
||||
"page_count": page,
|
||||
"result_count": 0,
|
||||
"http_status": status or 403,
|
||||
"blocked": True,
|
||||
"challenge": "captcha" if status in {401, 403} else None,
|
||||
"items": [],
|
||||
"discovery_mode": mode,
|
||||
}
|
||||
payload = parse_result_html(html, page_url=url)
|
||||
payload["page"] = page
|
||||
payload["http_status"] = status
|
||||
payload["discovery_mode"] = mode
|
||||
return payload
|
||||
|
||||
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if response.get("html") and not response.get("items"):
|
||||
return list(parse_result_html(str(response["html"]), page_url=str(response.get("url") or "")).get("items") or [])
|
||||
return list(response.get("items") or [])
|
||||
|
||||
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]:
|
||||
|
|
|
|||
104
backend/app/sources/geizhals/live_page.py
Normal file
104
backend/app/sources/geizhals/live_page.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.errors import AppError
|
||||
from app.sources.geizhals.taxonomy_parser import is_challenge_html
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
||||
BROWSER_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||
}
|
||||
EXPAND_LABELS = (
|
||||
"Immer alle Filter anzeigen",
|
||||
"weitere Filter anzeigen",
|
||||
)
|
||||
|
||||
|
||||
def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, int, bool]:
|
||||
status, html = 0, ""
|
||||
try:
|
||||
status, html = _fetch_http(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_http", status, False
|
||||
except Exception:
|
||||
status, html = 0, ""
|
||||
if playwright:
|
||||
try:
|
||||
status, html = _fetch_playwright(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_playwright", status, False
|
||||
except Exception:
|
||||
pass
|
||||
if html and not is_challenge_html(html) and status and status < 400:
|
||||
return html, "live_http", status, False
|
||||
return html, "blocked", status or 403, True
|
||||
|
||||
|
||||
def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]:
|
||||
status, html = 0, ""
|
||||
try:
|
||||
status, html = _fetch_http(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_http", status
|
||||
except Exception:
|
||||
status, html = 0, ""
|
||||
if playwright:
|
||||
try:
|
||||
status, html = _fetch_playwright(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_playwright", status
|
||||
except Exception:
|
||||
pass
|
||||
recorded = FIXTURE_DIR / f"{category_ref}-category.html"
|
||||
if recorded.exists():
|
||||
return recorded.read_text(encoding="utf-8"), "recorded_html", 0
|
||||
raise AppError(
|
||||
423,
|
||||
"SOURCE_BLOCKED",
|
||||
"Geizhals blockiert den Taxonomie-Abruf (403/Challenge)",
|
||||
{"url": url, "http_status": status},
|
||||
)
|
||||
|
||||
|
||||
def _fetch_http(url: str) -> tuple[int, str]:
|
||||
import httpx
|
||||
|
||||
response = httpx.get(url, headers=BROWSER_HEADERS, follow_redirects=True, timeout=25.0)
|
||||
return response.status_code, response.text
|
||||
|
||||
|
||||
def _fetch_playwright(url: str) -> tuple[int, str]:
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError as exc:
|
||||
raise AppError(
|
||||
423,
|
||||
"SOURCE_BLOCKED",
|
||||
"Geizhals verlangt einen Browser-Abruf, Playwright ist nicht installiert",
|
||||
) from exc
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(headless=True)
|
||||
page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS)
|
||||
response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||||
page.wait_for_timeout(4000)
|
||||
for label in EXPAND_LABELS:
|
||||
button = page.get_by_text(label, exact=False)
|
||||
if button.count():
|
||||
button.first.click(timeout=5000)
|
||||
page.wait_for_timeout(1500)
|
||||
html = page.content()
|
||||
status = response.status if response is not None else 200
|
||||
browser.close()
|
||||
return status, html
|
||||
|
||||
|
||||
def _blocked(status: int, html: str) -> bool:
|
||||
if status in {401, 403, 429}:
|
||||
return True
|
||||
return is_challenge_html(html)
|
||||
285
backend/app/sources/geizhals/result_parser.py
Normal file
285
backend/app/sources/geizhals/result_parser.py
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit
|
||||
|
||||
from app.sources.geizhals.taxonomy_parser import is_challenge_html, node_text, parse_html_tree
|
||||
|
||||
PRODUCT_ID_RE = re.compile(r"-a(\d+)\.html", re.IGNORECASE)
|
||||
PRICE_RE = re.compile(r"(\d{1,3}(?:\.\d{3})+|\d+),(\d{2})")
|
||||
OFFER_RE = re.compile(r"(\d+)\s*Angebote?", re.IGNORECASE)
|
||||
PRODUCT_COUNT_RE = re.compile(r"(\d[\d.]*)\s*Produkte", re.IGNORECASE)
|
||||
CAPACITY_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*(TB|GB)", re.IGNORECASE)
|
||||
WEIGHT_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*kg", re.IGNORECASE)
|
||||
AVAILABILITY_BY_CODE = {"0": "out_of_stock", "1": "limited", "2": "in_stock"}
|
||||
AVAILABILITY_BY_LABEL = {
|
||||
"lagernd": "in_stock",
|
||||
"kurzfristig lieferbar": "limited",
|
||||
"kurzfristig": "limited",
|
||||
"nicht verfügbar": "out_of_stock",
|
||||
}
|
||||
|
||||
|
||||
def parse_result_html(html: str, *, page_url: str) -> dict[str, Any]:
|
||||
if is_challenge_html(html):
|
||||
return {"items": [], "page_count": 1, "result_count": 0, "blocked": True}
|
||||
root = parse_html_tree(html)
|
||||
cards = _find_cards(root)
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for card in cards:
|
||||
item = _parse_card(card, page_url)
|
||||
if item is None or item["source_external_id"] in seen:
|
||||
continue
|
||||
seen.add(item["source_external_id"])
|
||||
items.append(item)
|
||||
page_count = _page_count(root)
|
||||
result_count = _result_count(root) or len(items)
|
||||
return {
|
||||
"items": items,
|
||||
"page_count": page_count,
|
||||
"result_count": result_count,
|
||||
"blocked": False,
|
||||
}
|
||||
|
||||
|
||||
def with_page(url: str, page: int) -> str:
|
||||
parts = urlsplit(url)
|
||||
query = [(name, value) for name, value in parse_qsl(parts.query, keep_blank_values=True) if name != "pg"]
|
||||
if page > 1:
|
||||
query.append(("pg", str(page)))
|
||||
return parts._replace(query=urlencode(query, doseq=True)).geturl()
|
||||
|
||||
|
||||
def _find_cards(root: Any) -> list[Any]:
|
||||
cards: list[Any] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
classes = node.attrs.get("class", "")
|
||||
if "galleryview__item" in classes or "listview__item" in classes:
|
||||
cards.append(node)
|
||||
return
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
walk(child)
|
||||
|
||||
walk(root)
|
||||
return cards
|
||||
|
||||
|
||||
def _parse_card(card: Any, page_url: str) -> dict[str, Any] | None:
|
||||
link = _first_link(card, "name-link") or _first_product_link(card)
|
||||
if link is None:
|
||||
return None
|
||||
href = link.attrs.get("href") or ""
|
||||
match = PRODUCT_ID_RE.search(href)
|
||||
if match is None:
|
||||
return None
|
||||
title = (link.attrs.get("title") or node_text(link)).strip()
|
||||
if not title:
|
||||
return None
|
||||
source_url = urljoin(page_url, href.split("#")[0])
|
||||
raw_attributes = _definition_list(card)
|
||||
sku = _first_text_by_class(card, "galleryview__mpn") or _first_text_by_class(card, "listview__mpn")
|
||||
price_text = _first_text_by_class(card, "price")
|
||||
offers_text = _first_text_by_class(card, "offercount") or ""
|
||||
availability_node = _first_by_attr(card, "data-availability")
|
||||
manufacturer, model = _split_title(title)
|
||||
return {
|
||||
"source_external_id": match.group(1),
|
||||
"source_url": source_url,
|
||||
"source_title": re.sub(r"\s+", " ", title),
|
||||
"manufacturer": manufacturer,
|
||||
"model": model,
|
||||
"manufacturer_sku": sku,
|
||||
"gtin_ean": None,
|
||||
"best_price": _parse_price(price_text),
|
||||
"currency": "EUR" if price_text and "€" in price_text else None,
|
||||
"availability": _availability(availability_node),
|
||||
"seller_count": _seller_count(offers_text),
|
||||
"raw_attributes": raw_attributes,
|
||||
"normalized_attributes": _normalize_attributes(raw_attributes),
|
||||
}
|
||||
|
||||
|
||||
def _first_link(node: Any, fragment: str) -> Any | None:
|
||||
classes = node.attrs.get("class", "")
|
||||
if node.tag == "a" and fragment in classes:
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_link(child, fragment)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _first_product_link(node: Any) -> Any | None:
|
||||
if node.tag == "a" and PRODUCT_ID_RE.search(node.attrs.get("href") or ""):
|
||||
href = node.attrs.get("href") or ""
|
||||
if "#ratings" not in href and "#offerlist" not in href:
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_product_link(child)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _first_text_by_class(node: Any, fragment: str) -> str | None:
|
||||
found = _first_by_class(node, fragment)
|
||||
if found is None:
|
||||
return None
|
||||
text = node_text(found).strip()
|
||||
return re.sub(r"\s+", " ", text) or None
|
||||
|
||||
|
||||
def _first_by_class(node: Any, fragment: str) -> Any | None:
|
||||
if fragment in node.attrs.get("class", ""):
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_by_class(child, fragment)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _first_by_attr(node: Any, name: str) -> Any | None:
|
||||
if name in node.attrs:
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_by_attr(child, name)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _definition_list(node: Any) -> dict[str, str]:
|
||||
pairs: dict[str, str] = {}
|
||||
|
||||
def walk(current: Any) -> None:
|
||||
if current.tag == "dl":
|
||||
pending: str | None = None
|
||||
for child in current.children:
|
||||
if isinstance(child, str):
|
||||
continue
|
||||
if child.tag == "dt":
|
||||
pending = re.sub(r"\s+", " ", node_text(child)).strip()
|
||||
elif child.tag == "dd" and pending:
|
||||
pairs[pending] = re.sub(r"\s+", " ", node_text(child)).strip()
|
||||
pending = None
|
||||
return
|
||||
for child in current.children:
|
||||
if not isinstance(child, str):
|
||||
walk(child)
|
||||
|
||||
walk(node)
|
||||
return pairs
|
||||
|
||||
|
||||
def _split_title(title: str) -> tuple[str | None, str | None]:
|
||||
parts = title.split(None, 1)
|
||||
if not parts:
|
||||
return None, None
|
||||
if len(parts) == 1:
|
||||
return parts[0], None
|
||||
return parts[0], parts[1]
|
||||
|
||||
|
||||
def _parse_price(text: str | None) -> str | None:
|
||||
if not text:
|
||||
return None
|
||||
match = PRICE_RE.search(text.replace("\xa0", " "))
|
||||
if match is None:
|
||||
return None
|
||||
whole = match.group(1).replace(".", "")
|
||||
return f"{whole}.{match.group(2)}"
|
||||
|
||||
|
||||
def _seller_count(text: str) -> int | None:
|
||||
match = OFFER_RE.search(text)
|
||||
if match is None:
|
||||
return None
|
||||
return int(match.group(1))
|
||||
|
||||
|
||||
def _availability(node: Any | None) -> str:
|
||||
if node is None:
|
||||
return "unknown"
|
||||
code = AVAILABILITY_BY_CODE.get(node.attrs.get("data-availability", ""))
|
||||
if code:
|
||||
return code
|
||||
label = (node.attrs.get("aria-label") or node_text(node)).strip().casefold()
|
||||
return AVAILABILITY_BY_LABEL.get(label, "unknown")
|
||||
|
||||
|
||||
def _normalize_attributes(raw: dict[str, str]) -> dict[str, Any]:
|
||||
normalized: dict[str, Any] = {}
|
||||
for label, value in raw.items():
|
||||
key = label.casefold()
|
||||
capacity = _capacity_gb(value)
|
||||
if capacity is not None and key in {"ram", "arbeitsspeicher"}:
|
||||
normalized["ram_gb"] = capacity
|
||||
elif capacity is not None and key in {"speicher", "ssd", "festplatte"}:
|
||||
normalized["storage_gb"] = capacity
|
||||
weight = _weight_kg(value)
|
||||
if weight is not None and "gewicht" in key:
|
||||
normalized["weight_kg"] = weight
|
||||
return normalized
|
||||
|
||||
|
||||
def _capacity_gb(text: str) -> float | int | None:
|
||||
match = CAPACITY_RE.search(text)
|
||||
if match is None:
|
||||
return None
|
||||
value = float(match.group(1).replace(",", "."))
|
||||
if match.group(2).upper() == "TB":
|
||||
value *= 1024
|
||||
return int(value) if value.is_integer() else value
|
||||
|
||||
|
||||
def _weight_kg(text: str) -> float | None:
|
||||
match = WEIGHT_RE.search(text)
|
||||
if match is None:
|
||||
return None
|
||||
return float(match.group(1).replace(",", "."))
|
||||
|
||||
|
||||
def _page_count(root: Any) -> int:
|
||||
pages = {1}
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
href = node.attrs.get("href") or ""
|
||||
if "pg=" in href and ("pagination" in node.attrs.get("class", "") or _in_pagination(node)):
|
||||
for name, value in parse_qsl(urlsplit(href).query):
|
||||
if name == "pg" and value.isdigit():
|
||||
pages.add(int(value))
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
walk(child)
|
||||
|
||||
walk(root)
|
||||
return max(pages)
|
||||
|
||||
|
||||
def _in_pagination(node: Any) -> bool:
|
||||
current = node
|
||||
while current is not None:
|
||||
classes = current.attrs.get("class", "")
|
||||
ident = current.attrs.get("id", "")
|
||||
if "pagination" in classes or ident == "pagination":
|
||||
return True
|
||||
current = current.parent
|
||||
return False
|
||||
|
||||
|
||||
def _result_count(root: Any) -> int | None:
|
||||
counts = [
|
||||
int(match.replace(".", ""))
|
||||
for match in PRODUCT_COUNT_RE.findall(node_text(root))
|
||||
]
|
||||
return max(counts) if counts else None
|
||||
474
backend/app/sources/geizhals/taxonomy_parser.py
Normal file
474
backend/app/sources/geizhals/taxonomy_parser.py
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, unquote, urlparse
|
||||
|
||||
from app.discovery.enums import FilterType
|
||||
|
||||
SKIP_OPTION_LABELS = {
|
||||
"alle",
|
||||
"entfernen",
|
||||
"überspringen",
|
||||
"aktualisieren",
|
||||
"abonnieren",
|
||||
"filter zurücksetzen",
|
||||
"nur beliebteste filter anzeigen",
|
||||
"immer alle filter anzeigen",
|
||||
"filter überspringen",
|
||||
"einfache filter",
|
||||
"erweiterte filter",
|
||||
}
|
||||
RANGE_MIN_NAMES = {"bpmin", "minp", "prmin"}
|
||||
RANGE_MAX_NAMES = {"bpmax", "maxp", "prmax"}
|
||||
TEXT_NAMES = {"asuch", "fs"}
|
||||
HEADING_TAGS = {"th", "h2", "h3", "h4", "h5", "legend", "dt", "caption", "label", "strong"}
|
||||
NUMBER_RE = re.compile(r"(-?\d+(?:[.,]\d+)?)")
|
||||
THRESHOLD_RE = re.compile(r"^(ab|bis)\s+", re.IGNORECASE)
|
||||
UNIT_RE = re.compile(r"(GB|TB|GHz|MHz|Hz|cd/m²|ppi|kg|\"|″|mm|%|€|EUR)$", re.IGNORECASE)
|
||||
TYPE_MAP = {
|
||||
"text": FilterType.ENUM.value,
|
||||
"color": FilterType.ENUM.value,
|
||||
"text_range": FilterType.NUMERIC_THRESHOLD.value,
|
||||
"numeric_range": FilterType.NUMERIC_RANGE.value,
|
||||
}
|
||||
|
||||
|
||||
class _Node:
|
||||
def __init__(self, tag: str, attrs: dict[str, str]) -> None:
|
||||
self.tag = tag
|
||||
self.attrs = attrs
|
||||
self.children: list[_Node | str] = []
|
||||
self.parent: _Node | None = None
|
||||
|
||||
|
||||
class _TreeBuilder(HTMLParser):
|
||||
VOID = {
|
||||
"area",
|
||||
"base",
|
||||
"br",
|
||||
"col",
|
||||
"embed",
|
||||
"hr",
|
||||
"img",
|
||||
"input",
|
||||
"link",
|
||||
"meta",
|
||||
"param",
|
||||
"source",
|
||||
"track",
|
||||
"wbr",
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = _Node("document", {})
|
||||
self._stack = [self.root]
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
node = _Node(tag, {key: value or "" for key, value in attrs})
|
||||
parent = self._stack[-1]
|
||||
node.parent = parent
|
||||
parent.children.append(node)
|
||||
if tag not in self.VOID:
|
||||
self._stack.append(node)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
for index in range(len(self._stack) - 1, 0, -1):
|
||||
if self._stack[index].tag == tag:
|
||||
del self._stack[index:]
|
||||
break
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
text = data.strip()
|
||||
if text:
|
||||
self._stack[-1].children.append(text)
|
||||
|
||||
|
||||
def parse_html_tree(html: str) -> _Node:
|
||||
builder = _TreeBuilder()
|
||||
builder.feed(html)
|
||||
return builder.root
|
||||
|
||||
|
||||
def node_text(node: _Node) -> str:
|
||||
return _node_text(node)
|
||||
|
||||
|
||||
def parse_taxonomy_html(html: str, *, category_ref: str, source_url: str) -> dict[str, Any]:
|
||||
if is_challenge_html(html):
|
||||
return {
|
||||
"source": "geizhals",
|
||||
"external_category_key": category_ref,
|
||||
"name": category_ref,
|
||||
"source_url": source_url,
|
||||
"filters": [],
|
||||
"blocked": True,
|
||||
}
|
||||
builder = _TreeBuilder()
|
||||
builder.feed(html)
|
||||
groups: dict[str, dict[str, Any]] = {}
|
||||
_collect_declared_filters(builder.root, groups)
|
||||
_collect_xf_links(builder.root, groups)
|
||||
_collect_named_inputs(builder.root, groups)
|
||||
filters = [
|
||||
group
|
||||
for group in groups.values()
|
||||
if _usable_group(group)
|
||||
]
|
||||
filters.sort(key=lambda item: (item["label"].lower(), item["external_filter_key"]))
|
||||
return {
|
||||
"source": "geizhals",
|
||||
"external_category_key": category_ref,
|
||||
"name": _category_name(builder.root, category_ref),
|
||||
"canonical_category": "notebook" if category_ref == "nb" else None,
|
||||
"source_url": source_url,
|
||||
"filters": filters,
|
||||
"blocked": False,
|
||||
}
|
||||
|
||||
|
||||
def is_challenge_html(html: str) -> bool:
|
||||
sample = html.lower()
|
||||
markers = (
|
||||
"cf-mitigated",
|
||||
"cf-challenge",
|
||||
"just a moment",
|
||||
"enable javascript and cookies",
|
||||
"checking your browser",
|
||||
"verification successful. waiting for geizhals",
|
||||
)
|
||||
return any(marker in sample for marker in markers)
|
||||
|
||||
|
||||
def _collect_declared_filters(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
|
||||
filter_id = node.attrs.get("data-filter-id")
|
||||
if filter_id:
|
||||
source_type = node.attrs.get("data-filter-type") or ""
|
||||
label = _declared_label(node) or f"Filter {filter_id}"
|
||||
group = groups.setdefault(filter_id, _empty_group(filter_id, label, source_type))
|
||||
if label and group["label"].startswith("Filter "):
|
||||
group["label"] = label
|
||||
if source_type:
|
||||
group["filter_type"] = TYPE_MAP.get(source_type, group["filter_type"])
|
||||
if source_type == "numeric_range":
|
||||
group["encoding"]["encoding_type"] = "RANGE_TOKEN"
|
||||
group["encoding"]["value_template"] = "{min}*{max}"
|
||||
group.setdefault("range", {"minimum": None, "maximum": None, "step": None, "unit": group.get("canonical_unit")})
|
||||
_collect_xf_links(node, groups, required_key=filter_id, allow_histogram=source_type == "numeric_range")
|
||||
return
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
_collect_declared_filters(child, groups)
|
||||
|
||||
|
||||
def _collect_xf_links(
|
||||
node: _Node,
|
||||
groups: dict[str, dict[str, Any]],
|
||||
required_key: str | None = None,
|
||||
allow_histogram: bool = False,
|
||||
) -> None:
|
||||
if node.tag == "a":
|
||||
href = node.attrs.get("href") or ""
|
||||
label = node.attrs.get("aria-label") or _node_text(node)
|
||||
if _usable_option(node, label, allow_histogram=allow_histogram):
|
||||
for key, query_value in _xf_tokens(href):
|
||||
if required_key is not None and key != required_key:
|
||||
continue
|
||||
if required_key is None and not _has_structural_label(node):
|
||||
continue
|
||||
group_label = _declared_label(_filter_root(node)) or _nearest_group_label(node)
|
||||
group = groups.setdefault(key, _empty_group(key, group_label, None))
|
||||
if group_label and (group["label"].startswith("Filter ") or group["label"] == key):
|
||||
group["label"] = group_label
|
||||
option = _option_from_link(label, query_value)
|
||||
existing = {item["source_value"] for item in group["options"]}
|
||||
if option["source_value"] not in existing:
|
||||
group["options"].append(option)
|
||||
group["encoding"].setdefault("options", {})
|
||||
group["encoding"]["options"][option["source_value"]] = query_value
|
||||
if option.get("normalized_value") is not None:
|
||||
group["encoding"]["options"][str(option["normalized_value"])] = query_value
|
||||
if group["filter_type"] not in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}:
|
||||
group["filter_type"] = _classify_options(group["options"])
|
||||
if group["canonical_unit"] is None:
|
||||
group["canonical_unit"] = option.get("unit")
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
_collect_xf_links(child, groups, required_key=required_key, allow_histogram=allow_histogram)
|
||||
|
||||
|
||||
def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
|
||||
inputs: list[_Node] = []
|
||||
_find_inputs(node, inputs)
|
||||
names = {item.attrs.get("name", ""): item for item in inputs if item.attrs.get("name")}
|
||||
if names.keys() & RANGE_MIN_NAMES or names.keys() & RANGE_MAX_NAMES:
|
||||
min_name = next((name for name in RANGE_MIN_NAMES if name in names), "bpmin")
|
||||
max_name = next((name for name in RANGE_MAX_NAMES if name in names), "bpmax")
|
||||
label = _nearest_group_label(names.get(min_name) or names.get(max_name) or node) or "Preis"
|
||||
groups["price"] = {
|
||||
"external_filter_key": "price",
|
||||
"label": "Preis" if "preis" not in label.lower() else label,
|
||||
"filter_type": FilterType.NUMERIC_RANGE.value,
|
||||
"canonical_unit": "EUR",
|
||||
"options": [],
|
||||
"range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"},
|
||||
"encoding": {
|
||||
"url_param": min_name,
|
||||
"min_parameter": min_name,
|
||||
"max_parameter": max_name,
|
||||
"encoding_type": "RANGE_PARAMETERS",
|
||||
},
|
||||
}
|
||||
for name in TEXT_NAMES:
|
||||
if name not in names:
|
||||
continue
|
||||
label = _nearest_group_label(names[name]) or "Suchbegriffe"
|
||||
groups[name] = {
|
||||
"external_filter_key": name,
|
||||
"label": label,
|
||||
"filter_type": FilterType.TEXT.value,
|
||||
"canonical_unit": None,
|
||||
"options": [],
|
||||
"encoding": {"url_param": name, "encoding_type": "TEXT_PARAMETER", "value_template": "{value}"},
|
||||
}
|
||||
|
||||
|
||||
def _find_inputs(node: _Node, found: list[_Node]) -> None:
|
||||
if node.tag == "input":
|
||||
found.append(node)
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
_find_inputs(child, found)
|
||||
|
||||
|
||||
def _xf_tokens(href: str) -> list[tuple[str, str]]:
|
||||
if not href or href.startswith("#") or href.startswith("javascript:"):
|
||||
return []
|
||||
query = urlparse(href).query
|
||||
tokens: list[tuple[str, str]] = []
|
||||
for name, value in parse_qsl(query, keep_blank_values=True):
|
||||
if name != "xf" or not value:
|
||||
continue
|
||||
for part in unquote(value).replace("|", "~").split("~"):
|
||||
if "_" not in part:
|
||||
continue
|
||||
key, raw = part.split("_", 1)
|
||||
if key.isdigit() and raw:
|
||||
tokens.append((key, raw.replace("+", " ")))
|
||||
return tokens
|
||||
|
||||
|
||||
def _option_from_link(label: str, query_value: str) -> dict[str, Any]:
|
||||
cleaned = re.sub(r"\(\d[\d.]*\s*(produkte)?\)\s*$", "", label, flags=re.IGNORECASE).strip()
|
||||
cleaned = re.sub(r"\s+", " ", cleaned)
|
||||
operator = None
|
||||
if cleaned.lower().startswith("ab "):
|
||||
operator = ">="
|
||||
elif cleaned.lower().startswith("bis "):
|
||||
operator = "<="
|
||||
number = _first_number(cleaned)
|
||||
unit = _unit(cleaned)
|
||||
return {
|
||||
"source_value": cleaned or query_value,
|
||||
"query_value": query_value,
|
||||
"normalized_value": number,
|
||||
"operator": operator,
|
||||
"unit": unit,
|
||||
}
|
||||
|
||||
|
||||
def _classify_options(options: list[dict[str, Any]]) -> str:
|
||||
if not options:
|
||||
return FilterType.UNKNOWN.value
|
||||
labels = [str(item["source_value"]).lower() for item in options]
|
||||
if all(item in {"ja", "nein", "egal", "true", "false"} for item in labels):
|
||||
return FilterType.BOOLEAN.value
|
||||
if all(THRESHOLD_RE.match(str(item["source_value"])) for item in options):
|
||||
return FilterType.NUMERIC_THRESHOLD.value
|
||||
return FilterType.ENUM.value
|
||||
|
||||
|
||||
def _empty_group(key: str, label: str | None, source_type: str | None) -> dict[str, Any]:
|
||||
encoding_type = "OPTION_TOKEN"
|
||||
if source_type == "numeric_range":
|
||||
encoding_type = "RANGE_TOKEN"
|
||||
return {
|
||||
"external_filter_key": key,
|
||||
"label": label or f"Filter {key}",
|
||||
"filter_type": TYPE_MAP.get(source_type or "", FilterType.ENUM.value),
|
||||
"canonical_unit": None,
|
||||
"options": [],
|
||||
"encoding": {
|
||||
"url_param": "xf",
|
||||
"encoding_type": encoding_type,
|
||||
"options": {},
|
||||
**({"value_template": "{min}*{max}"} if encoding_type == "RANGE_TOKEN" else {}),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _declared_label(node: _Node | None) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
attr_id = node.attrs.get("id", "")
|
||||
if attr_id.startswith("filter-element__title-label"):
|
||||
return _clean_label(_node_text(node)) or None
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
found = _declared_label(child)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _filter_root(node: _Node) -> _Node | None:
|
||||
current: _Node | None = node
|
||||
while current is not None:
|
||||
if current.attrs.get("data-filter-id"):
|
||||
return current
|
||||
current = current.parent
|
||||
return None
|
||||
|
||||
|
||||
def _nearest_group_label(node: _Node | None) -> str | None:
|
||||
current = node
|
||||
while current is not None:
|
||||
declared = _declared_label(current)
|
||||
if declared:
|
||||
return declared
|
||||
if current.tag in HEADING_TAGS:
|
||||
text = _clean_label(_node_text(current))
|
||||
if text:
|
||||
return text
|
||||
previous = _previous_element(current)
|
||||
while previous is not None:
|
||||
if previous.tag in HEADING_TAGS:
|
||||
text = _clean_label(_node_text(previous))
|
||||
if text:
|
||||
return text
|
||||
previous = _previous_element(previous)
|
||||
labeled = current.attrs.get("aria-label") or current.attrs.get("data-filter-name") or current.attrs.get("title")
|
||||
if labeled:
|
||||
text = _clean_label(labeled)
|
||||
if text and len(text.split()) <= 6:
|
||||
return text
|
||||
current = current.parent
|
||||
return None
|
||||
|
||||
|
||||
def _has_structural_label(node: _Node) -> bool:
|
||||
if _filter_root(node) is not None:
|
||||
return True
|
||||
current: _Node | None = node
|
||||
while current is not None:
|
||||
if current.tag in {"th", "td", "tr", "table", "dt", "dd"}:
|
||||
return _nearest_group_label(node) is not None
|
||||
current = current.parent
|
||||
return False
|
||||
|
||||
|
||||
def _usable_option(node: _Node, label: str, *, allow_histogram: bool) -> bool:
|
||||
classes = f"{node.attrs.get('class', '')} {node.attrs.get('id', '')}".lower()
|
||||
if "skip-link" in classes or "add-to-columns" in classes:
|
||||
return False
|
||||
if "bucket" in classes and not allow_histogram:
|
||||
return False
|
||||
cleaned = re.sub(r"\s+", " ", label).strip()
|
||||
if not cleaned or _skip_option(cleaned):
|
||||
return False
|
||||
if "überspringen" in cleaned.lower():
|
||||
return False
|
||||
if not allow_histogram and "produkte" in cleaned.lower():
|
||||
return False
|
||||
if len(cleaned.split()) > 8:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _usable_group(group: dict[str, Any]) -> bool:
|
||||
if group["filter_type"] == FilterType.TEXT.value:
|
||||
return True
|
||||
if group.get("range") and group["filter_type"] == FilterType.NUMERIC_RANGE.value:
|
||||
return True
|
||||
if not group.get("options") and not group.get("range"):
|
||||
return False
|
||||
label = str(group.get("label") or "")
|
||||
if label.startswith("Filter ") and len(group.get("options") or []) < 2:
|
||||
return False
|
||||
if len(label.split()) > 8:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _previous_element(node: _Node) -> _Node | None:
|
||||
if node.parent is None:
|
||||
return None
|
||||
siblings = [child for child in node.parent.children if isinstance(child, _Node)]
|
||||
try:
|
||||
index = siblings.index(node)
|
||||
except ValueError:
|
||||
return None
|
||||
return siblings[index - 1] if index > 0 else None
|
||||
|
||||
|
||||
def _node_text(node: _Node) -> str:
|
||||
parts: list[str] = []
|
||||
for child in node.children:
|
||||
if isinstance(child, str):
|
||||
parts.append(child)
|
||||
else:
|
||||
parts.append(_node_text(child))
|
||||
return " ".join(part for part in parts if part).strip()
|
||||
|
||||
|
||||
def _clean_label(text: str) -> str:
|
||||
cleaned = re.sub(r"\s+", " ", text).strip().rstrip(":")
|
||||
cleaned = re.sub(r"\s*(überspringen|entfernen)$", "", cleaned, flags=re.IGNORECASE).strip()
|
||||
if not cleaned or _skip_option(cleaned):
|
||||
return ""
|
||||
return cleaned
|
||||
|
||||
|
||||
def _skip_option(label: str) -> bool:
|
||||
return label.strip().lower() in SKIP_OPTION_LABELS
|
||||
|
||||
|
||||
def _first_number(text: str) -> float | int | None:
|
||||
match = NUMBER_RE.search(text.replace(",", "."))
|
||||
if match is None:
|
||||
return None
|
||||
raw = match.group(1)
|
||||
value = float(raw)
|
||||
return int(value) if value.is_integer() else value
|
||||
|
||||
|
||||
def _unit(text: str) -> str | None:
|
||||
match = UNIT_RE.search(text.replace(" ", ""))
|
||||
if match is None:
|
||||
if "kg" in text.lower():
|
||||
return "kg"
|
||||
return None
|
||||
unit = match.group(1)
|
||||
if unit in {'"', "″"}:
|
||||
return "inch"
|
||||
return unit
|
||||
|
||||
|
||||
def _category_name(root: _Node, fallback: str) -> str:
|
||||
titles: list[str] = []
|
||||
|
||||
def walk(node: _Node) -> None:
|
||||
if node.tag in {"h1", "title"}:
|
||||
text = _node_text(node)
|
||||
if text:
|
||||
titles.append(text)
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
walk(child)
|
||||
|
||||
walk(root)
|
||||
if titles:
|
||||
return titles[0].split("»")[0].strip() or fallback
|
||||
return "Notebooks" if fallback == "nb" else fallback
|
||||
132
backend/fixtures/geizhals/nb-category.html
Normal file
132
backend/fixtures/geizhals/nb-category.html
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<title>Notebooks Preisvergleich Geizhals Deutschland</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Notebooks</h1>
|
||||
<div data-filter-type="text_range" data-filter-id="17023">
|
||||
<span id="filter-element__title-label--17023">RAM</span>
|
||||
<a href="?cat=nb&xf=17023_ab+16GB&promode=true">ab 16GB</a>
|
||||
<a href="?cat=nb&xf=17023_ab+32GB&promode=true">ab 32GB</a>
|
||||
</div>
|
||||
<form>
|
||||
<label>Suchbegriffe</label>
|
||||
<input name="asuch" type="text">
|
||||
<label>Preisbereich (€)</label>
|
||||
<input name="bpmin" type="number" placeholder="Preis ab">
|
||||
<input name="bpmax" type="number" placeholder="Preis bis">
|
||||
</form>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Hersteller:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=1_Apple">Apple</a>
|
||||
<a href="?cat=nb&xf=1_ASUS">ASUS</a>
|
||||
<a href="?cat=nb&xf=1_Lenovo">Lenovo</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Display-Größe:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12128_13">13.x"</a>
|
||||
<a href="?cat=nb&xf=12128_14">14.x"</a>
|
||||
<a href="?cat=nb&xf=12128_15">15.x"</a>
|
||||
<a href="?cat=nb&xf=12128_16">16.x"</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Display-Auflösung:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=9_1920x1080">1920x1080 (Full HD)</a>
|
||||
<a href="?cat=nb&xf=9_2560x1600">2560x1600 (WQXGA)</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Kerne:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12822_8">8 (Octa-Core)</a>
|
||||
<a href="?cat=nb&xf=12822_10">10</a>
|
||||
<a href="?cat=nb&xf=12822_14">14-Core</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Kerne ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12822_8">ab 8</a>
|
||||
<a href="?cat=nb&xf=12822_10">ab 10</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Threads ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12823_12">ab 12</a>
|
||||
<a href="?cat=nb&xf=12823_16">ab 16</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Turbotakt:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=9682_4500">ab 4.5GHz</a>
|
||||
<a href="?cat=nb&xf=9682_5000">ab 5GHz</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>RAM ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12_16384">ab 16GB</a>
|
||||
<a href="?cat=nb&xf=12_32768">ab 32GB</a>
|
||||
<a href="?cat=nb&xf=12_65536">ab 64GB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>RAM-Typ:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=7520_DDR5">DDR5</a>
|
||||
<a href="?cat=nb&xf=7520_LPDDR5">LPDDR5</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Speichertyp:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=82_SSD">SSD</a>
|
||||
<a href="?cat=nb&xf=82_M.2">M.2</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>SSD-Kapazität:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=2991_1000">ab 1TB</a>
|
||||
<a href="?cat=nb&xf=2991_2000">ab 2TB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Grafik-Typ:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=8149_zzx">dedizierte Grafik (dGPU)</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>GPU - Speichergröße ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=7436_2048">ab 2GB</a>
|
||||
<a href="?cat=nb&xf=7436_4096">ab 4GB</a>
|
||||
<a href="?cat=nb&xf=7436_8192">ab 8GB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>GPU - Speichergröße:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=16692_8192">8GB</a>
|
||||
<a href="?cat=nb&xf=16692_12288">12GB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>USB-C mit DisplayPort gesamt (inkl. Thunderbolt):</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=13731_1">ab 1x</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
52
backend/fixtures/geizhals/nb-search-page.html
Normal file
52
backend/fixtures/geizhals/nb-search-page.html
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
<!doctype html>
|
||||
<html lang="de">
|
||||
<body>
|
||||
<a href="#productlist">6431 Produkte</a>
|
||||
<nav class="pagination">
|
||||
<a class="pagination__page" href="?cat=nb&pg=2">Seite 2</a>
|
||||
<a class="pagination__page last-page" href="?cat=nb&pg=4">Seite 4</a>
|
||||
</nav>
|
||||
<div class="galleryview__item card">
|
||||
<h3 class="galleryview__name">
|
||||
<a class="galleryview__name-link" href="acme-notebook-14-a1000001.html" title="Acme Notebook 14, 32GB RAM, 1TB SSD">
|
||||
Acme Notebook 14, 32GB RAM, 1TB SSD
|
||||
</a>
|
||||
</h3>
|
||||
<span class="galleryview__mpn">ACM-14-32</span>
|
||||
<dl>
|
||||
<dt>Betriebssystem</dt><dd>Linux</dd>
|
||||
<dt>Display</dt><dd>14"</dd>
|
||||
<dt>CPU</dt><dd>Acme 8-Core</dd>
|
||||
<dt>RAM</dt><dd>32GB</dd>
|
||||
<dt>Speicher</dt><dd>1TB SSD</dd>
|
||||
<dt>Grafik</dt><dd>Acme GPU</dd>
|
||||
</dl>
|
||||
<div class="galleryview__price">
|
||||
<a class="galleryview__price-link" href="acme-notebook-14-a1000001.html#offerlist">
|
||||
<span class="price">€ 1298,00</span>
|
||||
</a>
|
||||
<span class="availability" data-availability="2" aria-label="lagernd"></span>
|
||||
</div>
|
||||
<a class="galleryview__offercount-link">8 Angebote</a>
|
||||
</div>
|
||||
<div class="galleryview__item card">
|
||||
<h3 class="galleryview__name">
|
||||
<a class="galleryview__name-link" href="acme-notebook-13-a1000002.html" title="Acme Notebook 13, 8GB RAM">
|
||||
Acme Notebook 13, 8GB RAM
|
||||
</a>
|
||||
</h3>
|
||||
<span class="galleryview__mpn">ACM-13-8</span>
|
||||
<dl>
|
||||
<dt>RAM</dt><dd>8GB</dd>
|
||||
<dt>Speicher</dt><dd>512GB SSD</dd>
|
||||
</dl>
|
||||
<div class="galleryview__price">
|
||||
<a class="galleryview__price-link" href="acme-notebook-13-a1000002.html#offerlist">
|
||||
<span class="price">€ 899,00</span>
|
||||
</a>
|
||||
<span class="availability" data-availability="1" aria-label="kurzfristig lieferbar"></span>
|
||||
</div>
|
||||
<a class="galleryview__offercount-link">3 Angebote</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -6,6 +6,7 @@ requires-python = ">=3.13"
|
|||
dependencies = [
|
||||
"alembic>=1.16",
|
||||
"fastapi>=0.118",
|
||||
"httpx>=0.28",
|
||||
"psycopg[binary]>=3.2",
|
||||
"pydantic>=2.11",
|
||||
"pydantic-settings>=2.11",
|
||||
|
|
@ -18,6 +19,9 @@ test = [
|
|||
"httpx>=0.28",
|
||||
"pytest>=8.4",
|
||||
]
|
||||
live = [
|
||||
"playwright>=1.48",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=75"]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
|
@ -6,6 +7,8 @@ from uuid import UUID, uuid4
|
|||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("GEIZHALS_USE_FIXTURES", "1")
|
||||
|
||||
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
|
||||
from app.domain.models import (
|
||||
AttributeDefinition,
|
||||
|
|
@ -17,7 +20,17 @@ from app.domain.models import (
|
|||
)
|
||||
from app.domain.validation import parse_profile, parse_scoring_model
|
||||
|
||||
from app.settings import get_settings
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _geizhals_fixtures(monkeypatch):
|
||||
monkeypatch.setenv("GEIZHALS_USE_FIXTURES", "1")
|
||||
get_settings.cache_clear()
|
||||
yield
|
||||
get_settings.cache_clear()
|
||||
AS_OF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)
|
||||
CUTOFF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,33 @@ def test_broken_mapping_is_not_used():
|
|||
assert "ram_gb" in internal
|
||||
|
||||
|
||||
def test_multiple_xf_filters_join_with_tilde():
|
||||
url, _ = generate_search_url(
|
||||
category_url="https://geizhals.de/?cat=nb",
|
||||
criteria_items=[
|
||||
{"criterion": "ram_gb", "operator": ">=", "value": 16},
|
||||
{"criterion": "gpu", "operator": ">=", "value": 8},
|
||||
],
|
||||
mappings=[
|
||||
{
|
||||
"criterion_key": "ram_gb",
|
||||
"external_filter_key": "12",
|
||||
"source_value": "16384",
|
||||
"encoding": {"url_param": "xf"},
|
||||
"status": "VERIFIED",
|
||||
},
|
||||
{
|
||||
"criterion_key": "gpu",
|
||||
"external_filter_key": "7436",
|
||||
"source_value": "8192",
|
||||
"encoding": {"url_param": "xf"},
|
||||
"status": "VERIFIED",
|
||||
},
|
||||
],
|
||||
)
|
||||
assert "xf=12_16384~7436_8192" in url or "xf=7436_8192~12_16384" in url
|
||||
|
||||
|
||||
def test_roundtrip_verifies_xf_and_maxp():
|
||||
assert verify_roundtrip(
|
||||
category_url="https://geizhals.de/?cat=nb",
|
||||
|
|
|
|||
42
backend/tests/unit/test_geizhals_result_parser.py
Normal file
42
backend/tests/unit/test_geizhals_result_parser.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from pathlib import Path
|
||||
|
||||
from app.sources.geizhals.result_parser import parse_result_html, with_page
|
||||
|
||||
HTML = (Path(__file__).resolve().parents[2] / "fixtures" / "geizhals" / "nb-search-page.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def test_parser_reads_observed_gallery_cards() -> None:
|
||||
payload = parse_result_html(HTML, page_url="https://geizhals.de/?cat=nb")
|
||||
assert payload["blocked"] is False
|
||||
assert payload["page_count"] == 4
|
||||
assert payload["result_count"] == 6431
|
||||
assert [item["source_external_id"] for item in payload["items"]] == ["1000001", "1000002"]
|
||||
first = payload["items"][0]
|
||||
assert first["source_url"] == "https://geizhals.de/acme-notebook-14-a1000001.html"
|
||||
assert first["source_title"].startswith("Acme Notebook 14")
|
||||
assert first["manufacturer"] == "Acme"
|
||||
assert first["manufacturer_sku"] == "ACM-14-32"
|
||||
assert first["best_price"] == "1298.00"
|
||||
assert first["currency"] == "EUR"
|
||||
assert first["availability"] == "in_stock"
|
||||
assert first["seller_count"] == 8
|
||||
assert first["raw_attributes"]["RAM"] == "32GB"
|
||||
assert first["raw_attributes"]["Betriebssystem"] == "Linux"
|
||||
assert first["normalized_attributes"]["ram_gb"] == 32
|
||||
assert first["normalized_attributes"]["storage_gb"] == 1024
|
||||
|
||||
|
||||
def test_challenge_html_is_not_turned_into_candidates() -> None:
|
||||
payload = parse_result_html(
|
||||
"<html><body>Enable JavaScript and cookies to continue</body></html>",
|
||||
page_url="https://geizhals.de/?cat=nb",
|
||||
)
|
||||
assert payload["blocked"] is True
|
||||
assert payload["items"] == []
|
||||
|
||||
|
||||
def test_with_page_uses_geizhals_pg_parameter() -> None:
|
||||
assert with_page("https://geizhals.de/?cat=nb&xf=17023_ab+16GB", 1) == "https://geizhals.de/?cat=nb&xf=17023_ab+16GB"
|
||||
assert "pg=3" in with_page("https://geizhals.de/?cat=nb&xf=17023_ab+16GB", 3)
|
||||
76
backend/tests/unit/test_geizhals_taxonomy_parser.py
Normal file
76
backend/tests/unit/test_geizhals_taxonomy_parser.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
from pathlib import Path
|
||||
|
||||
from app.discovery.urlgen import generate_search_url
|
||||
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
|
||||
|
||||
HTML = (Path(__file__).resolve().parents[2] / "fixtures" / "geizhals" / "nb-category.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def test_parser_reads_href_filters_and_price_inputs() -> None:
|
||||
payload = parse_taxonomy_html(
|
||||
HTML,
|
||||
category_ref="nb",
|
||||
source_url="https://geizhals.de/?cat=nb",
|
||||
)
|
||||
keys = {item["external_filter_key"] for item in payload["filters"]}
|
||||
assert {"12", "17023", "price", "12128", "7436", "82", "asuch"} <= keys
|
||||
assert len(payload["filters"]) >= 12
|
||||
live_ram = next(item for item in payload["filters"] if item["external_filter_key"] == "17023")
|
||||
assert live_ram["label"] == "RAM"
|
||||
ram = next(item for item in payload["filters"] if item["external_filter_key"] == "12")
|
||||
assert ram["label"] == "RAM ab"
|
||||
assert ram["filter_type"] == "NUMERIC_THRESHOLD"
|
||||
assert any(option["source_value"] == "ab 16GB" for option in ram["options"])
|
||||
assert ram["encoding"]["options"]["ab 16GB"] == "16384"
|
||||
price = next(item for item in payload["filters"] if item["external_filter_key"] == "price")
|
||||
assert price["filter_type"] == "NUMERIC_RANGE"
|
||||
assert price["encoding"]["min_parameter"] == "bpmin"
|
||||
assert price["encoding"]["max_parameter"] == "bpmax"
|
||||
|
||||
|
||||
def test_learned_ram_and_price_encode_observed_geizhals_query() -> None:
|
||||
payload = parse_taxonomy_html(
|
||||
HTML,
|
||||
category_ref="nb",
|
||||
source_url="https://geizhals.de/?cat=nb",
|
||||
)
|
||||
ram = next(item for item in payload["filters"] if item["external_filter_key"] == "12")
|
||||
price = next(item for item in payload["filters"] if item["external_filter_key"] == "price")
|
||||
url, internal = generate_search_url(
|
||||
category_url="https://geizhals.de/?cat=nb",
|
||||
criteria_items=[
|
||||
{"criterion": "geizhals:nb:12", "operator": ">=", "value": 16},
|
||||
{"criterion": "geizhals:nb:price", "operator": "<=", "value": 1700},
|
||||
],
|
||||
mappings=[
|
||||
{
|
||||
"criterion_key": "geizhals:nb:12",
|
||||
"external_filter_key": "12",
|
||||
"source_value": "16384",
|
||||
"encoding": ram["encoding"],
|
||||
"status": "VERIFIED",
|
||||
},
|
||||
{
|
||||
"criterion_key": "geizhals:nb:price",
|
||||
"external_filter_key": "price",
|
||||
"source_value": None,
|
||||
"encoding": price["encoding"],
|
||||
"status": "VERIFIED",
|
||||
},
|
||||
],
|
||||
)
|
||||
assert internal == []
|
||||
assert "xf=12_16384" in url
|
||||
assert "bpmax=1700" in url
|
||||
|
||||
|
||||
def test_challenge_html_is_not_treated_as_taxonomy() -> None:
|
||||
payload = parse_taxonomy_html(
|
||||
"<html><body>Enable JavaScript and cookies to continue</body></html>",
|
||||
category_ref="nb",
|
||||
source_url="https://geizhals.de/?cat=nb",
|
||||
)
|
||||
assert payload["blocked"] is True
|
||||
assert payload["filters"] == []
|
||||
|
|
@ -12,3 +12,9 @@ def test_database_url_uses_psycopg_and_explicit_fields() -> None:
|
|||
assert settings.database_url == (
|
||||
"postgresql+psycopg://buyingagent:secret@postgres:5432/buyingagent_test"
|
||||
)
|
||||
|
||||
|
||||
def test_geizhals_live_defaults_are_off_for_fixtures_flag() -> None:
|
||||
settings = Settings(geizhals_use_fixtures=True, geizhals_playwright=False)
|
||||
assert settings.geizhals_use_fixtures is True
|
||||
assert settings.geizhals_playwright is False
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ source = { editable = "." }
|
|||
dependencies = [
|
||||
{ name = "alembic" },
|
||||
{ name = "fastapi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydantic-settings" },
|
||||
|
|
@ -62,6 +63,9 @@ dependencies = [
|
|||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
live = [
|
||||
{ name = "playwright" },
|
||||
]
|
||||
test = [
|
||||
{ name = "httpx" },
|
||||
{ name = "pytest" },
|
||||
|
|
@ -71,7 +75,9 @@ test = [
|
|||
requires-dist = [
|
||||
{ name = "alembic", specifier = ">=1.16" },
|
||||
{ name = "fastapi", specifier = ">=0.118" },
|
||||
{ name = "httpx", specifier = ">=0.28" },
|
||||
{ name = "httpx", marker = "extra == 'test'", specifier = ">=0.28" },
|
||||
{ name = "playwright", marker = "extra == 'live'", specifier = ">=1.48" },
|
||||
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2" },
|
||||
{ name = "pydantic", specifier = ">=2.11" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.11" },
|
||||
|
|
@ -79,7 +85,7 @@ requires-dist = [
|
|||
{ name = "sqlalchemy", specifier = ">=2.0" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.37" },
|
||||
]
|
||||
provides-extras = ["test"]
|
||||
provides-extras = ["test", "live"]
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
|
|
@ -125,6 +131,63 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "greenlet"
|
||||
version = "3.5.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/a1/e720a38852366c589e1a46cf570b886507ad2cf591050c203365638baab0/greenlet-3.5.6-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:f96f0e30b5a95c7631b12bfe214cbc90ec8fe8cfa36920596c10514a65743519", size = 294627, upload-time = "2026-09-14T14:24:40.102Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/c3/58187858df41354a11e6a55b421e7af9059798abdab3a384cc51b8567c38/greenlet-3.5.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c75116c9de79949de23006e2d9b35ee82874c594fcf5c0311b439acaa14b8441", size = 614356, upload-time = "2026-09-14T15:12:03.399Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/b9/3a7e67d5f05c9760b1ad411fa52264bd69cc08e22a2ebfb4018b90628ced/greenlet-3.5.6-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cad5782f93f7f738b62c6527b6f32a60694d924029f299a8b524758cfa53d815", size = 626756, upload-time = "2026-09-14T15:20:44.269Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/7c/40400455f5b5a65bb83e94fde66d1be9e5ec518638113f8083ace746c309/greenlet-3.5.6-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a93ee7c6e8fd0f8a83525a51bd777be57ee17787e91d805bd8d6faf9dcada18e", size = 632632, upload-time = "2026-09-14T15:25:07.813Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/cb/ab0c123c514ed4e94c0dc9ee2e86362633e6b998cfc05de7fc9ac2eb9690/greenlet-3.5.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f98e8215e172f567ce80eeaed9107fb4d32b6c44f26983d9b8334658136a205a", size = 623779, upload-time = "2026-09-14T14:36:01.104Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/67/1f35cff30a6c51c3f23b63d4afcc7313ab4f97490ba3676fa78178984b27/greenlet-3.5.6-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7f731ebac68ea06d628658295cb2d217b10186329fcf9a3b6a149045059bf92e", size = 434933, upload-time = "2026-09-14T15:28:38.858Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/26/fda8a5a06e7073333ccb038133c5893b9e0c4fe29d5992a17e83c241bc6e/greenlet-3.5.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df19e2d0b1620039af5102563fbd96e8938c7f5c3f5828528d641d9fc585525e", size = 1584930, upload-time = "2026-09-14T15:10:08.234Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2f/37/50f8813163148d6234e08b23dcad6a9e37f01d148c8ec976e4c44ea2d918/greenlet-3.5.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06c0e933290fba8ffe53ead4ae1b8044b0e9754b75cebf381aa2bc3e50d82fac", size = 1647590, upload-time = "2026-09-14T14:35:51.173Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/da/b7669b09586365654083a62bd0724cf06cb74bd5085a15cdd161271f992f/greenlet-3.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:5b602b4201b965a8354d74e232364a66ff243dd142e350d035f46169bb36e13d", size = 324086, upload-time = "2026-09-14T14:23:48.428Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/5d/c9663cfe84a2a9e0aa96f066f5b0594c227ea4c647511e087e2e11d4ac0a/greenlet-3.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:876077e7ebb8c84ed068e2b23d4c62ebb010d60df84b9591af1be2f39010ffb2", size = 308211, upload-time = "2026-09-14T14:28:01.634Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h11"
|
||||
version = "0.16.0"
|
||||
|
|
@ -340,6 +403,25 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "playwright"
|
||||
version = "1.63.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "greenlet" },
|
||||
{ name = "pyee" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/19/dd/fbb3d34228ad753bc14464d5f2252585b73367539775a88d0a3949cf5a45/playwright-1.63.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:84c540759e8e7f01e690e197e04060f48899d37cea322299f255843273d3385d", size = 44278407, upload-time = "2026-09-15T16:49:06.045Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/9a/948b930b1a8c4ee869e5a139a2b7747caa06aab56a3f09a2f0abdcbda221/playwright-1.63.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fd1aa00631d44d55e56e0975bf3f3f285fac4a9fd2813183f0c12a488f1a1b24", size = 42944679, upload-time = "2026-09-15T16:49:10.039Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/11/dc5c13fa1602371603acd461be47529c1b3513815d3a0dc98f642c291a10/playwright-1.63.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:c89fc4736502a1f0fac2c8ca5d10c0cbc1c669f1f4774a2d8507a43140e4d53f", size = 44278410, upload-time = "2026-09-15T16:49:13.861Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/9c/103a5037789062bdab27c7dca53f3ca6b075b572ab2cd96eec825b3aec4e/playwright-1.63.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ad21bc07516b187965a7521c5cf0df0bd657b17482eaad74335272d35a2b07de", size = 48217159, upload-time = "2026-09-15T16:49:17.404Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/82/3d85505284c5a210f2da6c07b8f757524e79d1fba9cfdafe1eafb766ae59/playwright-1.63.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:354e15b29503565fc598b89f16fbe070459343bef9d7498a93e304864000c6a7", size = 47902708, upload-time = "2026-09-15T16:49:21.055Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/69/8d/f74ff6b52751859f4b69caf66aa7d4a3c8d6c1f0c7dc9920da103612ee39/playwright-1.63.0-py3-none-win32.whl", hash = "sha256:660c00c62639e31b16700ba5456b351ddba55bb766b7ce8261223aa34928e482", size = 38606075, upload-time = "2026-09-15T16:49:29.546Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/eb/d6b8d92658038e260dbc7dd69fb3fdbe245aac283953de8b50b02bfe5f61/playwright-1.63.0-py3-none-win_amd64.whl", hash = "sha256:2f9a707a6c6c91157ed77bff2b8caeb04b3c8d46e70d585fc134298cbe4b5cc6", size = 38606082, upload-time = "2026-09-15T16:49:32.838Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/75/2432b3e3c7c62103b72d4c4cc8b16a56383ada372bbb0d1278591e988f71/playwright-1.63.0-py3-none-win_arm64.whl", hash = "sha256:1e4a3a838ce22fb68ad17193fcd142a19610d9d70fb9966d2239f5dddc0cc05b", size = 34523554, upload-time = "2026-09-15T16:49:36.455Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.6.0"
|
||||
|
|
@ -491,6 +573,18 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyee"
|
||||
version = "13.0.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.21.0"
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ services:
|
|||
ENVIRONMENT: "${ENVIRONMENT:-development}"
|
||||
ENGINE_BUILD: "${ENGINE_BUILD:-dev}"
|
||||
CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:5173,http://127.0.0.1:5173,http://192.168.2.49:5174}"
|
||||
GEIZHALS_USE_FIXTURES: "${GEIZHALS_USE_FIXTURES:-0}"
|
||||
GEIZHALS_PLAYWRIGHT: "${GEIZHALS_PLAYWRIGHT:-1}"
|
||||
ports:
|
||||
- "${BUYING_AGENT_API_PORT:-8095}:8000"
|
||||
depends_on:
|
||||
|
|
@ -84,6 +86,7 @@ services:
|
|||
ENVIRONMENT: test
|
||||
ENGINE_BUILD: test
|
||||
CLOCK_NOW: "2026-10-03T12:00:00Z"
|
||||
GEIZHALS_USE_FIXTURES: "1"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ services:
|
|||
ENVIRONMENT: test
|
||||
ENGINE_BUILD: test
|
||||
CLOCK_NOW: "2026-10-03T12:00:00Z"
|
||||
GEIZHALS_USE_FIXTURES: "1"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ services:
|
|||
ENVIRONMENT: "${ENVIRONMENT:-production}"
|
||||
ENGINE_BUILD: "${ENGINE_BUILD:-dev}"
|
||||
CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS:-http://localhost:4173,http://127.0.0.1:4173}"
|
||||
GEIZHALS_USE_FIXTURES: "${GEIZHALS_USE_FIXTURES:-0}"
|
||||
GEIZHALS_PLAYWRIGHT: "${GEIZHALS_PLAYWRIGHT:-1}"
|
||||
ports:
|
||||
- "${BUYING_AGENT_API_BIND:-127.0.0.1}:${BUYING_AGENT_API_PORT:-8000}:8000"
|
||||
depends_on:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ server {
|
|||
proxy_pass http://api:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_connect_timeout 10s;
|
||||
proxy_read_timeout 120s;
|
||||
proxy_send_timeout 120s;
|
||||
}
|
||||
|
||||
location /health {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
|||
});
|
||||
const body = await response.json().catch(() => ({}));
|
||||
if (!response.ok) {
|
||||
throw new Error(body.message || `HTTP ${response.status}`);
|
||||
throw new Error(body?.error?.message || body?.message || `HTTP ${response.status}`);
|
||||
}
|
||||
return body as T;
|
||||
}
|
||||
|
|
@ -101,7 +101,12 @@ export type AvailableCriterion = {
|
|||
supports_discovery: boolean;
|
||||
scoring_attribute_key?: string | null;
|
||||
};
|
||||
export type AvailableCriteria = { product_category_id: string; source: string; criteria: AvailableCriterion[] };
|
||||
export type AvailableCriteria = {
|
||||
product_category_id: string;
|
||||
source: string;
|
||||
taxonomy_status?: string | null;
|
||||
criteria: AvailableCriterion[];
|
||||
};
|
||||
export type DiscoverySearch = {
|
||||
id: string;
|
||||
status: string;
|
||||
|
|
|
|||
|
|
@ -33,9 +33,10 @@ export function AdminPage() {
|
|||
<p>Source: {source || "–"}</p>
|
||||
<p>Kategorie: {category || "–"}</p>
|
||||
<p>Status: {selected?.taxonomy_status || "NOT_LEARNED"}</p>
|
||||
<p>Gelernte Filter: {filters.data?.filters.length ?? 0}</p>
|
||||
{discover.isError ? <p className="danger">{(discover.error as Error).message}</p> : null}
|
||||
<button className="button" disabled={!source || !category || discover.isPending} onClick={() => discover.mutate()}>
|
||||
Taxonomie laden
|
||||
{discover.isPending ? "Lernt Taxonomie …" : "Taxonomie neu lernen"}
|
||||
</button>
|
||||
</div>
|
||||
{!category ? (
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ export function WizardPage() {
|
|||
const [sourceOverride, setSourceOverride] = useState<string | null>(null);
|
||||
const [selected, setSelected] = useState<string[]>([]);
|
||||
const [draft, setDraft] = useState<Draft>({});
|
||||
const [criterionQuery, setCriterionQuery] = useState("");
|
||||
|
||||
const sources = useQuery({
|
||||
queryKey: ["category-sources", categoryId],
|
||||
|
|
@ -63,7 +64,12 @@ export function WizardPage() {
|
|||
const sourceRow = sourceItems.find((item) => item.source === source) || sourceItems[0];
|
||||
const needsPicker = sourceItems.filter((item) => item.status === "ACTIVE").length > 1;
|
||||
|
||||
const visible = useMemo(() => criteria.data?.criteria || [], [criteria.data]);
|
||||
const visible = useMemo(() => {
|
||||
const items = criteria.data?.criteria || [];
|
||||
const query = criterionQuery.trim().toLowerCase();
|
||||
if (!query) return items;
|
||||
return items.filter((item) => item.label.toLowerCase().includes(query));
|
||||
}, [criteria.data, criterionQuery]);
|
||||
|
||||
if (categories.isLoading) return <LoadingState />;
|
||||
if (categories.isError) return <ErrorState message={(categories.error as Error).message} />;
|
||||
|
|
@ -84,6 +90,7 @@ export function WizardPage() {
|
|||
setCategoryId(event.target.value);
|
||||
setSourceOverride(null);
|
||||
setSelected([]);
|
||||
setCriterionQuery("");
|
||||
}}
|
||||
>
|
||||
<option value="">Bitte wählen</option>
|
||||
|
|
@ -139,10 +146,21 @@ export function WizardPage() {
|
|||
<p className="muted">Quelle: {source}</p>
|
||||
) : null}
|
||||
|
||||
{(criteria.data?.taxonomy_status || sourceRow?.taxonomy_status) === "DEGRADED" ? (
|
||||
<p className="muted">Die Taxonomie stammt aus einer aufgezeichneten Kategorieseite. Live-Lernen war blockiert.</p>
|
||||
) : null}
|
||||
{criteria.isError ? <ErrorState message={(criteria.error as Error).message} onRetry={() => criteria.refetch()} /> : null}
|
||||
{criteria.isLoading ? <LoadingState /> : null}
|
||||
{visible.length > 0 ? (
|
||||
{(criteria.data?.criteria || []).length > 0 ? (
|
||||
<div className="card grid">
|
||||
<p>
|
||||
{(criteria.data?.criteria || []).length} Kriterien aus der Quelle. Wähle die relevanten aus und setze
|
||||
Grenzwerte.
|
||||
</p>
|
||||
<label>
|
||||
Kriterien suchen
|
||||
<input value={criterionQuery} onChange={(event) => setCriterionQuery(event.target.value)} />
|
||||
</label>
|
||||
{visible.map((item) => {
|
||||
const on = selected.includes(item.criterion_definition_id);
|
||||
return (
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user