from __future__ import annotations import json from pathlib import Path from typing import Any from urllib.parse import parse_qs, urlparse from app.discovery.enums import FilterType from app.errors import AppError from app.settings import get_settings from app.sources.geizhals.category_url import parse_category_key, with_extended_filters from app.sources.geizhals.live_page import load_category_page, load_result_page from app.sources.geizhals.result_parser import parse_result_html, with_page from app.sources.geizhals.taxonomy_api import fetch_taxonomy_api from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals" class GeizhalsAdapter: source_key = "geizhals" def __init__(self, fixture_dir: Path | None = None, live: bool | None = None) -> None: self.fixture_dir = fixture_dir or FIXTURE_DIR settings = get_settings() self.live = (not settings.geizhals_use_fixtures) if live is None else live def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]: if not self.live: return self._load(f"{category_ref}-taxonomy.json") url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}") category_ref = parse_category_key(url) api_payload = fetch_taxonomy_api(source_url or url, category_ref=category_ref) if api_payload and api_payload.get("filters"): return api_payload html, mode, status = load_category_page( url, category_ref=category_ref, playwright=get_settings().geizhals_playwright, ) payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url) if payload.get("blocked") or not payload.get("filters"): raise AppError( 422, "TAXONOMY_EMPTY", "Geizhals-Taxonomie konnte nicht aus der Kategorieseite gelesen werden", {"url": url, "http_status": status, "mode": mode}, ) payload["discovery_mode"] = mode payload["http_status"] = status payload["taxonomy_status"] = "DEGRADED" if mode == "recorded_html" else "READY" return payload def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: from app.discovery.urlgen import generate_search_url category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None) category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}" url, internal_only = generate_search_url( category_url=category_url, criteria_items=criteria, mappings=mappings, ) return {"url": url, "internal_only": internal_only} def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]: page = int(pagination.get("page") or 1) if not self.live: category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0] return self._load(f"{category}-search-page{page}.json") url = with_page(str(query["url"]), page) html, mode, status, blocked = load_result_page(url, playwright=get_settings().geizhals_playwright) if blocked: return { "page": page, "page_count": page, "result_count": 0, "http_status": status or 403, "blocked": True, "challenge": "captcha" if status in {401, 403} else None, "items": [], "discovery_mode": mode, } payload = parse_result_html(html, page_url=url) payload["page"] = page payload["http_status"] = status payload["discovery_mode"] = mode return payload def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]: if response.get("html") and not response.get("items"): return list(parse_result_html(str(response["html"]), page_url=str(response.get("url") or "")).get("items") or []) return list(response.get("items") or []) def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]: attrs = result.get("normalized_attributes") or {} return { "source_external_id": result.get("source_external_id"), "source_url": result.get("source_url"), "source_title": result.get("source_title"), "manufacturer": result.get("manufacturer"), "model": result.get("model"), "manufacturer_sku": result.get("manufacturer_sku"), "gtin_ean": result.get("gtin_ean"), "best_price": result.get("best_price"), "currency": result.get("currency") or "EUR", "availability": result.get("availability") or "unknown", "seller_count": result.get("seller_count"), "raw_attributes": result.get("raw_attributes") or {}, "normalized_attributes": attrs, } def _load(self, name: str) -> dict[str, Any]: path = self.fixture_dir / name if not path.exists(): raise AppError(404, "FIXTURE_NOT_FOUND", f"geizhals fixture missing: {name}") return json.loads(path.read_text(encoding="utf-8")) def _category(self, mappings: list[dict[str, Any]], criteria: list[dict[str, Any]]) -> str: for mapping in mappings: if mapping.get("category_key"): return str(mapping["category_key"]) return "nb" def filter_type(raw: str) -> str: try: return FilterType(raw).value except ValueError: return FilterType.UNKNOWN.value