127 lines
5.5 KiB
Python
127 lines
5.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import parse_qs, urlparse
|
|
|
|
from app.discovery.enums import FilterType
|
|
from app.errors import AppError
|
|
from app.settings import get_settings
|
|
from app.sources.geizhals.category_url import parse_category_key, with_extended_filters
|
|
from app.sources.geizhals.live_page import load_category_page, load_result_page
|
|
from app.sources.geizhals.result_parser import parse_result_html, with_page
|
|
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
|
|
|
|
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
|
|
|
|
|
class GeizhalsAdapter:
|
|
source_key = "geizhals"
|
|
|
|
def __init__(self, fixture_dir: Path | None = None, live: bool | None = None) -> None:
|
|
self.fixture_dir = fixture_dir or FIXTURE_DIR
|
|
settings = get_settings()
|
|
self.live = (not settings.geizhals_use_fixtures) if live is None else live
|
|
|
|
def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]:
|
|
if not self.live:
|
|
return self._load(f"{category_ref}-taxonomy.json")
|
|
url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}")
|
|
category_ref = parse_category_key(url)
|
|
html, mode, status = load_category_page(
|
|
url,
|
|
category_ref=category_ref,
|
|
playwright=get_settings().geizhals_playwright,
|
|
)
|
|
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url)
|
|
if payload.get("blocked") or not payload.get("filters"):
|
|
raise AppError(
|
|
422,
|
|
"TAXONOMY_EMPTY",
|
|
"Geizhals-Taxonomie konnte nicht aus der Kategorieseite gelesen werden",
|
|
{"url": url, "http_status": status, "mode": mode},
|
|
)
|
|
payload["discovery_mode"] = mode
|
|
payload["http_status"] = status
|
|
payload["taxonomy_status"] = "DEGRADED" if mode == "recorded_html" else "READY"
|
|
return payload
|
|
|
|
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
|
|
from app.discovery.urlgen import generate_search_url
|
|
|
|
category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None)
|
|
category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
|
|
url, internal_only = generate_search_url(
|
|
category_url=category_url,
|
|
criteria_items=criteria,
|
|
mappings=mappings,
|
|
)
|
|
return {"url": url, "internal_only": internal_only}
|
|
|
|
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]:
|
|
page = int(pagination.get("page") or 1)
|
|
if not self.live:
|
|
category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0]
|
|
return self._load(f"{category}-search-page{page}.json")
|
|
url = with_page(str(query["url"]), page)
|
|
html, mode, status, blocked = load_result_page(url, playwright=get_settings().geizhals_playwright)
|
|
if blocked:
|
|
return {
|
|
"page": page,
|
|
"page_count": page,
|
|
"result_count": 0,
|
|
"http_status": status or 403,
|
|
"blocked": True,
|
|
"challenge": "captcha" if status in {401, 403} else None,
|
|
"items": [],
|
|
"discovery_mode": mode,
|
|
}
|
|
payload = parse_result_html(html, page_url=url)
|
|
payload["page"] = page
|
|
payload["http_status"] = status
|
|
payload["discovery_mode"] = mode
|
|
return payload
|
|
|
|
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]:
|
|
if response.get("html") and not response.get("items"):
|
|
return list(parse_result_html(str(response["html"]), page_url=str(response.get("url") or "")).get("items") or [])
|
|
return list(response.get("items") or [])
|
|
|
|
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]:
|
|
attrs = result.get("normalized_attributes") or {}
|
|
return {
|
|
"source_external_id": result.get("source_external_id"),
|
|
"source_url": result.get("source_url"),
|
|
"source_title": result.get("source_title"),
|
|
"manufacturer": result.get("manufacturer"),
|
|
"model": result.get("model"),
|
|
"manufacturer_sku": result.get("manufacturer_sku"),
|
|
"gtin_ean": result.get("gtin_ean"),
|
|
"best_price": result.get("best_price"),
|
|
"currency": result.get("currency") or "EUR",
|
|
"availability": result.get("availability") or "unknown",
|
|
"seller_count": result.get("seller_count"),
|
|
"raw_attributes": result.get("raw_attributes") or {},
|
|
"normalized_attributes": attrs,
|
|
}
|
|
|
|
def _load(self, name: str) -> dict[str, Any]:
|
|
path = self.fixture_dir / name
|
|
if not path.exists():
|
|
raise AppError(404, "FIXTURE_NOT_FOUND", f"geizhals fixture missing: {name}")
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
def _category(self, mappings: list[dict[str, Any]], criteria: list[dict[str, Any]]) -> str:
|
|
for mapping in mappings:
|
|
if mapping.get("category_key"):
|
|
return str(mapping["category_key"])
|
|
return "nb"
|
|
|
|
|
|
def filter_type(raw: str) -> str:
|
|
try:
|
|
return FilterType(raw).value
|
|
except ValueError:
|
|
return FilterType.UNKNOWN.value
|