from __future__ import annotations from typing import Any from uuid import uuid4 from sqlalchemy import select from sqlalchemy.orm import Session from app.discovery.enums import FilterType, MappingStatus, TaxonomyStatus from app.domain.serialize import jsonable from app.errors import AppError from app.persistence.discovery_models import ( CriterionDefinitionRecord, CriterionMappingRecord, ProductCategoryRecord, SourceCategoryRecord, SourceFilterDefinitionRecord, SourceFilterOptionRecord, SourceFilterRangeRecord, ) from app.discovery.definitions import upsert_definition from app.persistence.repositories import CatalogRepository FILTER_OPERATORS = { FilterType.BOOLEAN.value: ["=", "!="], FilterType.ENUM.value: ["=", "!="], FilterType.MULTI_ENUM.value: ["in", "not_in"], FilterType.NUMERIC_RANGE.value: [">=", "<=", "between"], FilterType.NUMERIC_THRESHOLD.value: [">=", ">", "<=", "<"], FilterType.TEXT.value: ["=", "contains"], } class CatalogService: def __init__(self, session: Session) -> None: self.session = session self.catalog = CatalogRepository(session) def list_product_categories(self) -> list[dict[str, Any]]: rows = self.session.scalars(select(ProductCategoryRecord).order_by(ProductCategoryRecord.name)).all() return [{"id": row.id, "name": row.name, "status": row.status} for row in rows] def get_product_category(self, category_id: str) -> ProductCategoryRecord: row = self.session.get(ProductCategoryRecord, category_id) if row is None: raise AppError(404, "NOT_FOUND", "product category not found", {"id": category_id}) return row def sources_for_category(self, product_category_id: str) -> list[dict[str, Any]]: self.get_product_category(product_category_id) rows = self.session.scalars( select(SourceCategoryRecord) .where(SourceCategoryRecord.product_category_id == product_category_id) .order_by(SourceCategoryRecord.source_id) ).all() sources = {item.id: item for item in self.catalog.list_sources()} return [ { "source": row.source_id, "source_category_id": row.id, "display_name": sources[row.source_id].name if row.source_id in sources else row.name, "external_category_key": row.external_category_key, "status": row.status, "taxonomy_status": row.taxonomy_status, } for row in rows ] def register_source_category(self, source_id: str, payload: dict[str, Any]) -> dict[str, Any]: if self.catalog.get_source(source_id) is None: raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) from app.sources.geizhals.category_url import parse_category_key, slugify from app.services.discovery_service import TaxonomyService source_url = str(payload["source_url"]).strip() category_key = parse_category_key(source_url) name = str(payload.get("name") or category_key) product_category_id = str(payload.get("product_category_id") or slugify(name)) ensure_product_category(self.session, product_category_id, str(payload.get("product_category_name") or name)) category = ensure_source_category( self.session, source_id=source_id, external_category_key=category_key, product_category_id=product_category_id, name=name, source_url=source_url, ) category.source_url = source_url category.name = name category.product_category_id = product_category_id self.session.flush() TaxonomyService(self.session).discover(source_id, category_key) items = self.source_categories(source_id) return next(item for item in items if item["external_category_key"] == category_key) def source_categories(self, source_id: str) -> list[dict[str, Any]]: if self.catalog.get_source(source_id) is None: raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) rows = self.session.scalars( select(SourceCategoryRecord) .where(SourceCategoryRecord.source_id == source_id) .order_by(SourceCategoryRecord.external_category_key) ).all() return [ { "id": row.id, "source": row.source_id, "source_category_id": row.id, "external_category_key": row.external_category_key, "name": row.name, "product_category_id": row.product_category_id, "status": row.status, "taxonomy_status": row.taxonomy_status, "source_url": row.source_url, } for row in rows ] def available_criteria(self, product_category_id: str, source_id: str) -> dict[str, Any]: self.get_product_category(product_category_id) if self.catalog.get_source(source_id) is None: raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) source_category = self.session.scalars( select(SourceCategoryRecord).where( SourceCategoryRecord.source_id == source_id, SourceCategoryRecord.product_category_id == product_category_id, ) ).first() if source_category is not None and self._should_learn(source_category, source_id): from app.services.discovery_service import TaxonomyService TaxonomyService(self.session).discover(source_id, source_category.external_category_key) source_category = self.session.scalars( select(SourceCategoryRecord).where( SourceCategoryRecord.source_id == source_id, SourceCategoryRecord.product_category_id == product_category_id, ) ).first() criteria: list[dict[str, Any]] = [] if source_category is not None: filters = self.session.scalars( select(SourceFilterDefinitionRecord) .where(SourceFilterDefinitionRecord.source_category_id == source_category.id) .order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label) ).all() mappings = { row.external_filter_key: row for row in self.session.scalars( select(CriterionMappingRecord).where( CriterionMappingRecord.source_category_id == source_category.id ) ) } for filt in filters: mapping = mappings.get(filt.external_filter_key) definition = None if mapping is None else self.session.get(CriterionDefinitionRecord, mapping.criterion_definition_id) if definition is None: definition = upsert_definition( self.session, { "key": f"{source_id}:{source_category.external_category_key}:{filt.external_filter_key}", "origin": "SOURCE", "source": source_id, "product_category_id": product_category_id, "label": filt.label, "value_type": "number" if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value} else "string", "unit": filt.canonical_unit, "category": product_category_id, }, ) self.session.flush() options = self.session.scalars( select(SourceFilterOptionRecord) .where(SourceFilterOptionRecord.filter_id == filt.id) .order_by(SourceFilterOptionRecord.source_value) ).all() rng = self.session.get(SourceFilterRangeRecord, filt.id) criteria.append( self._criterion_out( definition, filt, mapping, options, rng, ) ) internals = self.session.scalars( select(CriterionDefinitionRecord) .where( CriterionDefinitionRecord.origin == "INTERNAL", (CriterionDefinitionRecord.product_category_id == product_category_id) | (CriterionDefinitionRecord.category == product_category_id), ) .order_by(CriterionDefinitionRecord.label) ).all() seen = {item["criterion_definition_id"] for item in criteria} for definition in internals: if definition.id in seen: continue criteria.append(self._criterion_out(definition, None, None, [], None)) return jsonable( { "product_category_id": product_category_id, "source": source_id, "taxonomy_status": None if source_category is None else source_category.taxonomy_status, "criteria": criteria, } ) def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool: if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value: return True if source_category.taxonomy_status != TaxonomyStatus.READY.value: return False from app.settings import get_settings if get_settings().geizhals_use_fixtures or source_id != "geizhals": return False keys = { row.external_filter_key for row in self.session.scalars( select(SourceFilterDefinitionRecord).where( SourceFilterDefinitionRecord.source_category_id == source_category.id ) ) } return keys <= {"17023", "price"} def list_templates(self) -> list[dict[str, Any]]: rows, _total = self.catalog.list_profiles(200, 0) return [ { "id": row.id, "version": row.version, "name": row.definition.get("name") or row.id, "product_category_id": row.definition.get("category"), "market": row.definition.get("market"), "currency": row.definition.get("currency"), "description": row.definition.get("description"), } for row in rows ] def _criterion_out( self, definition: CriterionDefinitionRecord, filt: SourceFilterDefinitionRecord | None, mapping: CriterionMappingRecord | None, options: list[SourceFilterOptionRecord], rng: SourceFilterRangeRecord | None, ) -> dict[str, Any]: filter_type = None if filt is None else filt.filter_type return { "criterion_definition_id": definition.id, "label": definition.label, "group_label": None if filt is None else (filt.group_label or "Allgemein"), "description": definition.description, "origin": definition.origin, "source": definition.source, "product_category_id": definition.product_category_id, "filter_type": filter_type, "unit": definition.unit or (None if filt is None else filt.canonical_unit), "operators": FILTER_OPERATORS.get(filter_type or "", ["=", ">=", "<="]), "options": [ { "source_value": option.source_value, "normalized_value": option.normalized_value, "operator": option.operator, "unit": option.unit, } for option in options ], "range": None if rng is None else {"minimum": rng.minimum, "maximum": rng.maximum, "step": rng.step, "unit": rng.unit}, "mapping_status": None if mapping is None else mapping.status, "supports_discovery": mapping is not None and mapping.status == MappingStatus.VERIFIED.value and filt is not None, "scoring_attribute_key": definition.scoring_attribute_key, } def ensure_product_category(session: Session, category_id: str, name: str | None = None) -> ProductCategoryRecord: row = session.get(ProductCategoryRecord, category_id) if row is not None: return row row = ProductCategoryRecord(id=category_id, name=name or category_id.title(), status="ACTIVE") session.add(row) session.flush() return row def ensure_source_category( session: Session, *, source_id: str, external_category_key: str, product_category_id: str, name: str, source_url: str, ) -> SourceCategoryRecord: row = session.scalars( select(SourceCategoryRecord).where( SourceCategoryRecord.source_id == source_id, SourceCategoryRecord.external_category_key == external_category_key, ) ).first() if row is None: row = SourceCategoryRecord( id=uuid4(), source_id=source_id, external_category_key=external_category_key, name=name, source_url=source_url, product_category_id=product_category_id, canonical_category=product_category_id, status="ACTIVE", taxonomy_status=TaxonomyStatus.NOT_LEARNED.value, last_taxonomy_scan_at=None, last_verified_at=None, ) session.add(row) session.flush() return row if row.product_category_id is None: row.product_category_id = product_category_id if not row.source_url: row.source_url = source_url session.flush() return row