diff --git a/.gitea/workflows/deploy-dev.yml b/.gitea/workflows/deploy-dev.yml index 66c026e..7604abd 100644 --- a/.gitea/workflows/deploy-dev.yml +++ b/.gitea/workflows/deploy-dev.yml @@ -47,7 +47,7 @@ jobs: if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then echo 'BUYING_AGENT_FRONTEND_PORT=5174' >> .env fi - docker compose -f docker-compose.dev-env.yml build --pull=false api frontend + COMPOSE_BAKE=0 docker compose -f docker-compose.dev-env.yml build --pull=false api frontend if ! ENGINE_BUILD="$ENGINE_BUILD" BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \ docker compose -f docker-compose.dev-env.yml up -d --wait; then echo "compose up --wait failed — api logs:" diff --git a/.gitea/workflows/deploy-prod.yml b/.gitea/workflows/deploy-prod.yml index d9505b0..069687b 100644 --- a/.gitea/workflows/deploy-prod.yml +++ b/.gitea/workflows/deploy-prod.yml @@ -54,7 +54,7 @@ jobs: if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then echo 'BUYING_AGENT_FRONTEND_PORT=4173' >> .env fi - docker compose build --pull=false api frontend + COMPOSE_BAKE=0 docker compose build --pull=false api frontend if ! ENGINE_BUILD="$ENGINE_BUILD" \ BUYING_AGENT_API_BIND="$BUYING_AGENT_API_BIND" \ BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \ diff --git a/backend/Dockerfile b/backend/Dockerfile index 7a8c461..0f0790b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -19,7 +19,7 @@ FROM base AS runtime COPY . . ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright -RUN uv sync --frozen --extra live --no-dev \ +RUN uv sync --frozen --extra live --extra test --no-dev \ && playwright install --with-deps chromium \ && sed -i 's/\r$//' /app/scripts/startup.sh \ && chmod +x /app/scripts/startup.sh \ diff --git a/backend/app/discovery/urlgen.py b/backend/app/discovery/urlgen.py index 5b73888..ad5b795 100644 --- a/backend/app/discovery/urlgen.py +++ b/backend/app/discovery/urlgen.py @@ -51,9 +51,7 @@ def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[ if encoding.get("encoding_type") == "RANGE_TOKEN": return _encode_range_token(mapping, criterion) if param == "xf": - filter_key = str(mapping["external_filter_key"]) - source_value = _xf_value(mapping, criterion) - return [("xf", f"{filter_key}_{_plus(source_value)}")] + return _encode_xf(mapping, criterion) if param in {"minp", "maxp", "prmin", "prmax", "asuch", "fs"}: return [(param, str(criterion.get("value")))] template = encoding.get("value_template") @@ -112,10 +110,30 @@ def _lookup_option(mapping: dict[str, Any], value: Any) -> str | None: return None +def _encode_xf(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]: + filter_key = str(mapping["external_filter_key"]) + raw = criterion.get("value") + values = raw if isinstance(raw, list) else [raw] + fragments: list[tuple[str, str]] = [] + for value in values: + if value is None or value == "": + continue + source_value = _xf_value(mapping, {**criterion, "value": value}) + fragments.append(("xf", f"{filter_key}_{_plus(source_value)}")) + if fragments: + return fragments + source_value = _xf_value(mapping, criterion) + return [("xf", f"{filter_key}_{_plus(source_value)}")] + + def _xf_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str: looked_up = _lookup_option(mapping, criterion.get("value")) if looked_up is not None: return looked_up + options = (mapping.get("encoding") or {}).get("options") or {} + value = criterion.get("value") + if options and value is not None and value != "": + return str(value) return str(mapping.get("source_value") or _threshold_value(mapping, criterion)) diff --git a/backend/app/persistence/discovery_models.py b/backend/app/persistence/discovery_models.py index 55c0922..6554894 100644 --- a/backend/app/persistence/discovery_models.py +++ b/backend/app/persistence/discovery_models.py @@ -118,6 +118,7 @@ class SourceFilterDefinitionRecord(Base): source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False) external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False) label: Mapped[str] = mapped_column(String(256), nullable=False) + group_label: Mapped[str | None] = mapped_column(String(128)) filter_type: Mapped[str] = mapped_column(String(32), nullable=False) canonical_unit: Mapped[str | None] = mapped_column(String(32)) status: Mapped[str] = mapped_column(String(32), nullable=False) diff --git a/backend/app/services/catalog_service.py b/backend/app/services/catalog_service.py index 8c6e3f6..27d46d6 100644 --- a/backend/app/services/catalog_service.py +++ b/backend/app/services/catalog_service.py @@ -142,7 +142,7 @@ class CatalogService: filters = self.session.scalars( select(SourceFilterDefinitionRecord) .where(SourceFilterDefinitionRecord.source_category_id == source_category.id) - .order_by(SourceFilterDefinitionRecord.label) + .order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label) ).all() mappings = { row.external_filter_key: row @@ -173,7 +173,9 @@ class CatalogService: ) self.session.flush() options = self.session.scalars( - select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id) + select(SourceFilterOptionRecord) + .where(SourceFilterOptionRecord.filter_id == filt.id) + .order_by(SourceFilterOptionRecord.source_value) ).all() rng = self.session.get(SourceFilterRangeRecord, filt.id) criteria.append( @@ -254,6 +256,7 @@ class CatalogService: return { "criterion_definition_id": definition.id, "label": definition.label, + "group_label": None if filt is None else (filt.group_label or "Allgemein"), "description": definition.description, "origin": definition.origin, "source": definition.source, diff --git a/backend/app/services/discovery_service.py b/backend/app/services/discovery_service.py index 1f2c16d..14957b1 100644 --- a/backend/app/services/discovery_service.py +++ b/backend/app/services/discovery_service.py @@ -171,6 +171,7 @@ class TaxonomyService: source_category_id=category.id, external_filter_key=str(raw["external_filter_key"]), label=str(raw.get("label") or raw["external_filter_key"]), + group_label=raw.get("group_label") or "Allgemein", filter_type=filter_type, canonical_unit=raw.get("canonical_unit"), status=MappingStatus.LEARNED_UNVERIFIED.value, @@ -181,6 +182,7 @@ class TaxonomyService: if filt.filter_type != filter_type: self._break_mappings(category.id, filt.external_filter_key) filt.label = str(raw.get("label") or filt.label) + filt.group_label = raw.get("group_label") or filt.group_label or "Allgemein" filt.filter_type = filter_type filt.canonical_unit = raw.get("canonical_unit") self._sync_options(filt, raw.get("options") or []) @@ -223,12 +225,14 @@ class TaxonomyService: rows = self.session.scalars( select(SourceFilterDefinitionRecord) .where(SourceFilterDefinitionRecord.source_category_id == category.id) - .order_by(SourceFilterDefinitionRecord.external_filter_key) + .order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label) ).all() filters = [] for filt in rows: options = self.session.scalars( - select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id) + select(SourceFilterOptionRecord) + .where(SourceFilterOptionRecord.filter_id == filt.id) + .order_by(SourceFilterOptionRecord.source_value) ).all() rng = self.session.get(SourceFilterRangeRecord, filt.id) filters.append( @@ -236,6 +240,7 @@ class TaxonomyService: "id": filt.id, "external_filter_key": filt.external_filter_key, "label": filt.label, + "group_label": filt.group_label or "Allgemein", "filter_type": filt.filter_type, "canonical_unit": filt.canonical_unit, "status": filt.status, @@ -912,7 +917,7 @@ class SearchExecutionService: pages: list[dict[str, Any]] = [] items: list[dict[str, Any]] = [] try: - pages, items, error_code = self._paginate(adapter, search) + pages, items, error_code, advertised = self._paginate(adapter, search) except AppError as exc: run.status = RunStatus.FAILED.value run.error_code = exc.code @@ -921,7 +926,7 @@ class SearchExecutionService: self.session.flush() return self.searches.get_run(run.id) run.page_count = len(pages) - run.result_count = len(items) + run.result_count = max(len(items), advertised) run.http_status_summary = {"pages": pages} run.error_code = error_code run.finished_at = now_utc() @@ -957,33 +962,38 @@ class SearchExecutionService: self, adapter: DiscoverySourceAdapter, search: DiscoverySearchRecord, - ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None]: + ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None, int]: query = {"url": search.generated_url} collected: list[dict[str, Any]] = [] pages: list[dict[str, Any]] = [] page = 1 expected_pages = 1 + advertised = 0 while page <= expected_pages and page <= MAX_PAGES: try: response = adapter.fetch_results(query, {"page": page}) except AppError as exc: if exc.code == "FIXTURE_NOT_FOUND" and page > 1: - return pages, collected, "PARTIAL" + return pages, collected, "PARTIAL", advertised raise status = int(response.get("http_status") or 200) - pages.append({"page": page, "status": status}) + pages.append({"page": page, "status": status, "result_count": response.get("result_count")}) if status == 429: - return pages, collected, "BACKOFF" + return pages, collected, "BACKOFF", advertised if status == 403 or response.get("challenge") == "captcha" or response.get("blocked"): - return pages, collected, "BLOCKED" + return pages, collected, "BLOCKED", advertised if status >= 400: - return pages, collected, "PARTIAL" + return pages, collected, "PARTIAL", advertised collected.extend(adapter.parse_results(response)) + if response.get("result_count") is not None: + advertised = max(advertised, int(response["result_count"])) + if advertised > search.broad_result_threshold: + return pages, collected, None, advertised expected_pages = int(response.get("page_count") or page) page += 1 if page > MAX_PAGES + 1 or (expected_pages > MAX_PAGES and len(pages) >= MAX_PAGES): - return pages, collected, "PARTIAL" - return pages, collected, None + return pages, collected, "PARTIAL", advertised + return pages, collected, None, advertised def _backoff(self, search_id: UUID) -> timedelta: rows = self.session.scalars( @@ -1407,7 +1417,7 @@ class CandidateRankingService: "source_url": None if ref is None else ref.source_url, "current_best_price": None if obs is None else obs.best_price, "currency": None if obs is None else obs.currency, - "current_dynamic_attributes": {} if obs is None else _dynamic_attributes(self.session, obs.id), + "current_dynamic_attributes": [] if obs is None else _dynamic_attributes(self.session, obs.id), "product_score": row.product_score, "product_eligibility": row.product_eligibility, "ranking_status": ranking_status, diff --git a/backend/app/sources/geizhals/live_page.py b/backend/app/sources/geizhals/live_page.py index a860ae9..83522bb 100644 --- a/backend/app/sources/geizhals/live_page.py +++ b/backend/app/sources/geizhals/live_page.py @@ -20,6 +20,7 @@ EXPAND_LABELS = ( "Immer alle Filter anzeigen", "weitere Filter anzeigen", ) +MORE_OPTION_SELECTOR = 'button:has-text("weitere"), a:has-text("weitere")' def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, int, bool]: @@ -101,6 +102,7 @@ def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, s continue if expand_filters: _expand_filter_sections(page) + _expand_more_options(page) html = page.content() status = response.status if response is not None else 200 browser.close() @@ -119,6 +121,18 @@ def _expand_filter_sections(page: object) -> None: return +def _expand_more_options(page: object) -> None: + locator = page.locator(MORE_OPTION_SELECTOR) + for _ in range(80): + if locator.count() == 0: + return + try: + locator.first.click(timeout=2000) + page.wait_for_timeout(350) + except Exception: + return + + def _blocked(status: int, html: str) -> bool: if status in {401, 403, 429}: return True diff --git a/backend/app/sources/geizhals/taxonomy_parser.py b/backend/app/sources/geizhals/taxonomy_parser.py index 6c91659..71df612 100644 --- a/backend/app/sources/geizhals/taxonomy_parser.py +++ b/backend/app/sources/geizhals/taxonomy_parser.py @@ -19,7 +19,26 @@ SKIP_OPTION_LABELS = { "filter überspringen", "einfache filter", "erweiterte filter", + "weitere", + "weniger", + "alle anzeigen", + "weniger anzeigen", } +PRODUCT_COUNT_RE = re.compile(r"\(\d[\d.]*\s*(produkte)?\)\s*$", re.IGNORECASE) +OPTION_TAGS = {"a", "button", "label", "input", "span"} +GROUP_ORDER = ( + "Allgemein", + "Display", + "CPU", + "RAM", + "RAM erweitert", + "Grafik", + "Anschlüsse", + "Wireless", + "Akku", + "Abmessungen", + "Sonstiges", +) RANGE_MIN_NAMES = {"bpmin", "minp", "prmin"} RANGE_MAX_NAMES = {"bpmax", "maxp", "prmax"} TEXT_NAMES = {"asuch", "fs"} @@ -117,7 +136,11 @@ def parse_taxonomy_html(html: str, *, category_ref: str, source_url: str) -> dic for group in groups.values() if _usable_group(group) ] - filters.sort(key=lambda item: (item["label"].lower(), item["external_filter_key"])) + for group in filters: + section = group.get("group_label") or "Allgemein" + group["group_label"] = section + group["group_sort"] = _group_sort(section) + filters.sort(key=lambda item: (item["group_sort"], item["group_label"].lower(), item["label"].lower(), item["external_filter_key"])) return { "source": "geizhals", "external_category_key": category_ref, @@ -148,6 +171,9 @@ def _collect_declared_filters(node: _Node, groups: dict[str, dict[str, Any]]) -> source_type = node.attrs.get("data-filter-type") or "" label = _declared_label(node) or f"Filter {filter_id}" group = groups.setdefault(filter_id, _empty_group(filter_id, label, source_type)) + section = _section_label(node) + if section: + group["group_label"] = section if label and group["label"].startswith("Filter "): group["label"] = label if source_type: @@ -169,19 +195,21 @@ def _collect_xf_links( required_key: str | None = None, allow_histogram: bool = False, ) -> None: - if node.tag == "a": - href = node.attrs.get("href") or "" - label = node.attrs.get("aria-label") or _node_text(node) + if node.tag in OPTION_TAGS: + label = node.attrs.get("aria-label") or node.attrs.get("title") or _node_text(node) if _usable_option(node, label, allow_histogram=allow_histogram): - for key, query_value in _xf_tokens(href): + for key, query_value in _node_xf_tokens(node): if required_key is not None and key != required_key: continue if required_key is None and not _has_structural_label(node): continue - group_label = _declared_label(_filter_root(node)) or _nearest_group_label(node) - group = groups.setdefault(key, _empty_group(key, group_label, None)) - if group_label and (group["label"].startswith("Filter ") or group["label"] == key): - group["label"] = group_label + filter_label = _declared_label(_filter_root(node)) or _nearest_group_label(node) + group = groups.setdefault(key, _empty_group(key, filter_label, None)) + section = _section_label(node) + if section: + group["group_label"] = section + if filter_label and (group["label"].startswith("Filter ") or group["label"] == key): + group["label"] = filter_label option = _option_from_link(label, query_value) existing = {item["source_value"] for item in group["options"]} if option["source_value"] not in existing: @@ -207,11 +235,14 @@ def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> Non min_name = next((name for name in RANGE_MIN_NAMES if name in names), "bpmin") max_name = next((name for name in RANGE_MAX_NAMES if name in names), "bpmax") label = _nearest_group_label(names.get(min_name) or names.get(max_name) or node) or "Preis" + section = _section_label(names.get(min_name) or names.get(max_name) or node) or "Allgemein" groups["price"] = { "external_filter_key": "price", "label": "Preis" if "preis" not in label.lower() else label, "filter_type": FilterType.NUMERIC_RANGE.value, "canonical_unit": "EUR", + "group_label": section, + "group_sort": _group_sort(section), "options": [], "range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"}, "encoding": { @@ -225,11 +256,14 @@ def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> Non if name not in names: continue label = _nearest_group_label(names[name]) or "Suchbegriffe" + section = _section_label(names[name]) or "Allgemein" groups[name] = { "external_filter_key": name, "label": label, "filter_type": FilterType.TEXT.value, "canonical_unit": None, + "group_label": section, + "group_sort": _group_sort(section), "options": [], "encoding": {"url_param": name, "encoding_type": "TEXT_PARAMETER", "value_template": "{value}"}, } @@ -243,6 +277,30 @@ def _find_inputs(node: _Node, found: list[_Node]) -> None: _find_inputs(child, found) +def _node_xf_tokens(node: _Node) -> list[tuple[str, str]]: + tokens = _xf_tokens(node.attrs.get("href") or "") + if tokens: + return tokens + for attr in ("data-xf", "data-filter-value", "data-value", "value"): + raw = node.attrs.get(attr) or "" + if attr == "value" and node.attrs.get("name") not in {"xf", ""} and node.tag != "input": + continue + parsed = _xf_raw_token(raw) + if parsed: + return [parsed] + return [] + + +def _xf_raw_token(raw: str) -> tuple[str, str] | None: + value = unquote(raw).replace("|", "~").split("~")[0] + if "_" not in value: + return None + key, token = value.split("_", 1) + if key.isdigit() and token: + return key, token.replace("+", " ") + return None + + def _xf_tokens(href: str) -> list[tuple[str, str]]: if not href or href.startswith("#") or href.startswith("javascript:"): return [] @@ -261,8 +319,7 @@ def _xf_tokens(href: str) -> list[tuple[str, str]]: def _option_from_link(label: str, query_value: str) -> dict[str, Any]: - cleaned = re.sub(r"\(\d[\d.]*\s*(produkte)?\)\s*$", "", label, flags=re.IGNORECASE).strip() - cleaned = re.sub(r"\s+", " ", cleaned) + cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip() operator = None if cleaned.lower().startswith("ab "): operator = ">=" @@ -299,6 +356,8 @@ def _empty_group(key: str, label: str | None, source_type: str | None) -> dict[s "label": label or f"Filter {key}", "filter_type": TYPE_MAP.get(source_type or "", FilterType.ENUM.value), "canonical_unit": None, + "group_label": None, + "group_sort": _group_sort("Allgemein"), "options": [], "encoding": { "url_param": "xf", @@ -375,18 +434,62 @@ def _usable_option(node: _Node, label: str, *, allow_histogram: bool) -> bool: return False if "bucket" in classes and not allow_histogram: return False - cleaned = re.sub(r"\s+", " ", label).strip() + cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip() if not cleaned or _skip_option(cleaned): return False - if "überspringen" in cleaned.lower(): + if "überspringen" in cleaned.lower() or "aufklappen" in cleaned.lower() or "zuklappen" in cleaned.lower(): return False - if not allow_histogram and "produkte" in cleaned.lower(): + if cleaned.lower().startswith("weitere"): return False - if len(cleaned.split()) > 8: + if not allow_histogram and re.fullmatch(r"\d[\d.]*\s*produkte", cleaned, re.IGNORECASE): + return False + if len(cleaned.split()) > 12: return False return True +def _section_label(node: _Node | None) -> str | None: + current = node + while current is not None: + classes = current.attrs.get("class", "").lower() + if "filter-row" in classes or current.attrs.get("data-filter-row-id"): + text = _strip_section_suffix( + current.attrs.get("data-filter-row-name") + or current.attrs.get("aria-label") + or _row_heading(current) + ) + if text: + return text + labeled = current.attrs.get("aria-label") or "" + if re.search(r"(auf|zu)klappen", labeled, re.IGNORECASE): + text = _strip_section_suffix(labeled) + if text: + return text + current = current.parent + return None + + +def _row_heading(node: _Node) -> str: + for child in node.children: + if isinstance(child, _Node) and child.tag in {"button", "h2", "h3", "h4", "legend", "strong"}: + text = _clean_label(_node_text(child)) + if text: + return text + return _clean_label(node.attrs.get("aria-label") or "") + + +def _strip_section_suffix(text: str | None) -> str: + cleaned = _clean_label(text or "") + return re.sub(r"\s*(aufklappen|zuklappen|einklappen)$", "", cleaned, flags=re.IGNORECASE).strip() + + +def _group_sort(label: str) -> int: + try: + return GROUP_ORDER.index(label) + except ValueError: + return len(GROUP_ORDER) + + def _usable_group(group: dict[str, Any]) -> bool: if group["filter_type"] == FilterType.TEXT.value: return True diff --git a/backend/fixtures/geizhals/nb-category.html b/backend/fixtures/geizhals/nb-category.html index c7cc3bc..fc67075 100644 --- a/backend/fixtures/geizhals/nb-category.html +++ b/backend/fixtures/geizhals/nb-category.html @@ -5,11 +5,20 @@

Notebooks

+
+
RAM ab 16GB ab 32GB
+
+ Hersteller + Apple + ASUS + + +
@@ -17,15 +26,10 @@
+
+
+ - - - -
Hersteller: - Apple - ASUS - Lenovo -
Display-Größe: @@ -128,5 +132,6 @@
+
diff --git a/backend/migrations/versions/0007_filter_groups.py b/backend/migrations/versions/0007_filter_groups.py new file mode 100644 index 0000000..548f331 --- /dev/null +++ b/backend/migrations/versions/0007_filter_groups.py @@ -0,0 +1,24 @@ +"""Persist source filter groups from the learned taxonomy. + +Revision ID: 0007_filter_groups +Revises: 0006_source_category_readiness +Create Date: 2026-10-04 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0007_filter_groups" +down_revision: str | None = "0006_source_category_readiness" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("source_filter_definitions", sa.Column("group_label", sa.String(128))) + + +def downgrade() -> None: + op.drop_column("source_filter_definitions", "group_label") diff --git a/backend/tests/unit/test_discovery_urlgen.py b/backend/tests/unit/test_discovery_urlgen.py index 9483fb8..4d9d522 100644 --- a/backend/tests/unit/test_discovery_urlgen.py +++ b/backend/tests/unit/test_discovery_urlgen.py @@ -95,6 +95,24 @@ def test_multiple_xf_filters_join_with_tilde(): assert "xf=12_16384~7436_8192" in url or "xf=7436_8192~12_16384" in url +def test_enum_uses_selected_option_not_first_mapping_value(): + url, _ = generate_search_url( + category_url="https://geizhals.de/?cat=nb", + criteria_items=[{"criterion": "hersteller", "operator": "=", "value": "Lenovo"}], + mappings=[ + { + "criterion_key": "hersteller", + "external_filter_key": "1", + "source_value": "Apple", + "encoding": {"url_param": "xf", "options": {"Apple": "Apple", "Lenovo": "Lenovo"}}, + "status": "VERIFIED", + } + ], + ) + assert "xf=1_Lenovo" in url + assert "Apple" not in url + + def test_roundtrip_verifies_xf_and_maxp(): assert verify_roundtrip( category_url="https://geizhals.de/?cat=nb", diff --git a/backend/tests/unit/test_geizhals_taxonomy_parser.py b/backend/tests/unit/test_geizhals_taxonomy_parser.py index 1561736..079c34b 100644 --- a/backend/tests/unit/test_geizhals_taxonomy_parser.py +++ b/backend/tests/unit/test_geizhals_taxonomy_parser.py @@ -28,6 +28,13 @@ def test_parser_reads_href_filters_and_price_inputs() -> None: assert price["filter_type"] == "NUMERIC_RANGE" assert price["encoding"]["min_parameter"] == "bpmin" assert price["encoding"]["max_parameter"] == "bpmax" + hersteller = next(item for item in payload["filters"] if item["external_filter_key"] == "1") + assert hersteller["label"] == "Hersteller" + assert hersteller["filter_type"] == "ENUM" + assert hersteller["group_label"] == "Allgemein" + assert {option["source_value"] for option in hersteller["options"]} >= {"Apple", "ASUS", "Lenovo"} + display = next(item for item in payload["filters"] if item["external_filter_key"] == "12128") + assert display["group_label"] == "Display" def test_learned_ram_and_price_encode_observed_geizhals_query() -> None: diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index aeda9bb..699bff8 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -67,9 +67,9 @@ Watchtower bleibt aus. | `deploy-prod.yml` | Push `main` | Deploy nach `~/docker/buyingagent`, Health `localhost:8006` | | `test.yml` | nach Dev-Deploy bzw. Push `develop` | pytest auf `buyingagent_test` (Dev-Postgres, nicht Live-DB) und read-only Smoke gegen `/health/live` und `/health/ready` | -Der API-Build nutzt `ghcr.io` (nicht Docker Hub). `docker compose build --pull=false` verwendet vorhandene Basis-Images und vermeidet IPv6-Timeouts gegen `auth.docker.io`. +Der API-Build nutzt `ghcr.io` (nicht Docker Hub). `COMPOSE_BAKE=0 docker compose build --pull=false` verwendet vorhandene Basis-Images und vermeidet Registry-Metadatenabrufe, die auf dem Pi per IPv6-DNS timeouten. -`test.yml` folgt Kanshō: pytest läuft in einem Einweg-Container (`api-test`, Compose-Profil `test`) gegen die wegwerfbare Datenbank `buyingagent_test` auf dem bereits laufenden Dev-Postgres. Die Live-Datenbank bleibt unangetastet. Es gibt kein Frontend, daher keinen Frontend-Build-Job. +`test.yml` folgt Kanshō: pytest läuft in einem Einweg-Container aus dem bereits deployedten `api`-Image gegen die wegwerfbare Datenbank `buyingagent_test` auf dem bereits laufenden Dev-Postgres. Die Live-Datenbank bleibt unangetastet. Es gibt kein Frontend, daher keinen Frontend-Build-Job. --- diff --git a/frontend/src/api.ts b/frontend/src/api.ts index fd3b297..17c9e7c 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -94,6 +94,7 @@ export type ProjectCriterion = { export type AvailableCriterion = { criterion_definition_id: string; label: string; + group_label?: string | null; origin: string; filter_type?: string | null; unit?: string | null; @@ -170,4 +171,11 @@ export type SourceCategory = { product_category_id?: string | null; source_url?: string | null; }; -export type SourceFilter = { label: string; external_filter_key: string; filter_type: string; status: string }; +export type SourceFilter = { + label: string; + group_label?: string | null; + external_filter_key: string; + filter_type: string; + status: string; + options?: { source_value: string; normalized_value: unknown }[]; +}; diff --git a/frontend/src/components/CriteriaControl.test.tsx b/frontend/src/components/CriteriaControl.test.tsx index ec83dc2..45fc746 100644 --- a/frontend/src/components/CriteriaControl.test.tsx +++ b/frontend/src/components/CriteriaControl.test.tsx @@ -22,6 +22,28 @@ test("renders numeric threshold without notebook-specific fields", () => { expect(screen.queryByLabelText("RAM GB")).not.toBeInTheDocument(); }); +test("renders enum field list from learned options", () => { + render( + undefined} + />, + ); + expect(screen.getByRole("option", { name: "Apple" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Lenovo" })).toBeInTheDocument(); +}); + test("picks a single active source automatically", () => { expect(pickSource([{ source: "geizhals", status: "ACTIVE" }])).toBe("geizhals"); expect( diff --git a/frontend/src/components/CriteriaControl.tsx b/frontend/src/components/CriteriaControl.tsx index b5000ab..4f5fe5e 100644 --- a/frontend/src/components/CriteriaControl.tsx +++ b/frontend/src/components/CriteriaControl.tsx @@ -12,6 +12,9 @@ export function CriteriaControl({ onChange: (next: { operator: string; value: unknown }) => void; }) { const type = criterion.filter_type || "TEXT"; + const options = criterion.options || []; + const optionValue = (option: { source_value: string; normalized_value: unknown }) => + String(option.normalized_value ?? option.source_value); return (
- ) : type === "ENUM" || type === "MULTI_ENUM" ? ( + ) : type === "MULTI_ENUM" ? ( +
+ Werte + {options.map((option) => { + const selected = Array.isArray(value) ? value.map(String).includes(optionValue(option)) : false; + return ( + + ); + })} +
+ ) : options.length > 0 && (type === "ENUM" || type === "NUMERIC_THRESHOLD") ? ( - {visible.map((item) => { - const on = selected.includes(item.criterion_definition_id); - return ( -
- - {on ? ( - ="} - value={draft[item.criterion_definition_id]?.value} - onChange={(next) => setDraft((current) => ({ ...current, [item.criterion_definition_id]: next }))} - /> - ) : null} -
- ); - })} + {grouped.map(([group, items]) => ( +
+ {group} + {items.map((item) => { + const on = selected.includes(item.criterion_definition_id); + return ( +
+ + {on ? ( + ="} + value={draft[item.criterion_definition_id]?.value} + onChange={(next) => setDraft((current) => ({ ...current, [item.criterion_definition_id]: next }))} + /> + ) : null} +
+ ); + })} +
+ ))}
) : null} {create.isError ? : null} ); diff --git a/frontend/src/styles.css b/frontend/src/styles.css index ea3f133..52dd849 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -28,6 +28,10 @@ th, td { text-align: left; padding: 8px 6px; border-bottom: 1px solid #2a2d36; } .button.secondary { background: #2a2d36; } input, select { background: #111318; color: #e8eaed; border: 1px solid #3a3f4b; border-radius: 8px; padding: 8px; width: 100%; } label { display: grid; gap: 6px; font-size: 14px; } +.criterion-group { border: 1px solid #2a2d36; border-radius: 10px; padding: 12px; display: grid; gap: 10px; } +.criterion-group legend { padding: 0 6px; color: #c5cad3; } +.option-list { margin: 8px 0 0; padding-left: 18px; } +.option-item { display: flex; align-items: center; gap: 8px; } .tabs { display: flex; gap: 8px; margin-bottom: 16px; } .tabs a { padding: 6px 10px; border-radius: 8px; background: #181b22; } .tabs a.active { background: #1d4ed8; } diff --git a/scripts/ci-pytest.sh b/scripts/ci-pytest.sh index 9f1ea58..d095b1d 100644 --- a/scripts/ci-pytest.sh +++ b/scripts/ci-pytest.sh @@ -41,24 +41,20 @@ docker compose -f "$COMPOSE_FILE" exec -T postgres \ psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \ -c "CREATE DATABASE buyingagent_test OWNER ${PGUSER};" -docker compose -f "$COMPOSE_FILE" --profile test build --pull=false api-test +# Reuse the already-deployed api image. A fresh api-test build hits ghcr.io +# metadata even with --pull=false and fails on IPv6 DNS timeouts. +run_test() { + docker compose -f "$COMPOSE_FILE" run --rm --no-deps --entrypoint "" \ + -v "${APP_DIR}:/src:ro" \ + -w /src/backend \ + -e PYTHONPATH=/src/backend \ + -e DB_NAME=buyingagent_test \ + -e ENVIRONMENT=test \ + -e ENGINE_BUILD=test \ + -e CLOCK_NOW=2026-10-03T12:00:00Z \ + -e GEIZHALS_USE_FIXTURES=1 \ + api "$@" +} -docker compose -f "$COMPOSE_FILE" --profile test run --rm --no-deps \ - -v "${APP_DIR}:/src:ro" \ - -w /src/backend \ - -e PYTHONPATH=/src/backend \ - -e DB_NAME=buyingagent_test \ - -e ENVIRONMENT=test \ - -e ENGINE_BUILD=test \ - -e CLOCK_NOW=2026-10-03T12:00:00Z \ - api-test /app/.venv/bin/alembic upgrade head - -docker compose -f "$COMPOSE_FILE" --profile test run --rm --no-deps \ - -v "${APP_DIR}:/src:ro" \ - -w /src/backend \ - -e PYTHONPATH=/src/backend \ - -e DB_NAME=buyingagent_test \ - -e ENVIRONMENT=test \ - -e ENGINE_BUILD=test \ - -e CLOCK_NOW=2026-10-03T12:00:00Z \ - api-test /app/.venv/bin/pytest tests -m "not slow" -ra --tb=short +run_test /app/.venv/bin/alembic upgrade head +run_test /app/.venv/bin/pytest tests -m "not slow" -ra --tb=short