from __future__ import annotations from datetime import timedelta from decimal import Decimal from typing import Any from uuid import UUID, uuid4 from sqlalchemy import select from sqlalchemy.orm import Session from app.clock import now_utc from app.discovery.definitions import get_definition, get_definition_by_key, upsert_definition from app.discovery.diff import diff_run, next_lifecycle from app.discovery.enums import ( CandidateStatus, DiffStatus, FilterType, LifecycleStatus, MappingStatus, MatchStatus, ProjectStatus, RankingStatus, RunStatus, SearchStatus, TaxonomyStatus, ) from app.discovery.identity import IdentitySignals, decide_identity, spec_fingerprint from app.discovery.mapping import verify_roundtrip from app.discovery.ranking import rank_candidates from app.discovery.urlgen import generate_search_url from app.domain.serialize import jsonable from app.domain.validation import canonical_json, sha256_text from app.errors import AppError from app.persistence.discovery_models import ( BuyingProjectRecord, CandidateStatusHistoryRecord, CriterionDefinitionRecord, CriterionMappingRecord, DiscoverySearchRecord, ProductCategoryRecord, ProductObservationAttributeRecord, ProductObservationRecord, ProjectCandidateRecord, SearchRunItemRecord, SearchRunRecord, SourceCategoryRecord, SourceFilterDefinitionRecord, SourceFilterOptionRecord, SourceFilterRangeRecord, SourceAttributeMappingRecord, SourceProductReferenceRecord, ) from app.persistence.repositories import CatalogRepository from app.services.catalog_service import ensure_product_category from app.services.project_service import ProjectService from app.sources.base import DiscoverySourceAdapter from app.sources.geizhals.adapter import GeizhalsAdapter MAX_PAGES = 50 BACKOFF_BASE_MINUTES = 15 def discovery_adapter(source_id: str) -> DiscoverySourceAdapter: if source_id == "geizhals": return GeizhalsAdapter() raise AppError(422, "UNKNOWN_SOURCE", "no discovery adapter for source", {"source_id": source_id}) def _linked_product_category(session: Session, canonical: str | None) -> str | None: if not canonical: return None category = session.get(ProductCategoryRecord, canonical) if category is None: ensure_product_category(session, canonical) return canonical def mapping_version_hash(mappings: list[CriterionMappingRecord]) -> str: payload = [ { "criterion_key": row.criterion_key, "external_filter_key": row.external_filter_key, "source_value": row.source_value, "encoding": row.encoding, } for row in mappings if row.status == MappingStatus.VERIFIED.value ] payload.sort(key=lambda item: (item["criterion_key"], item["external_filter_key"], str(item["source_value"]))) return sha256_text(canonical_json(payload)) class TaxonomyService: def __init__(self, session: Session, adapter: DiscoverySourceAdapter | None = None) -> None: self.session = session self.adapter = adapter def discover(self, source_id: str, category_key: str) -> dict[str, Any]: catalog = CatalogRepository(self.session) if catalog.get_source(source_id) is None: raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id}) adapter = self.adapter or discovery_adapter(source_id) existing = self.session.scalars( select(SourceCategoryRecord).where( SourceCategoryRecord.source_id == source_id, SourceCategoryRecord.external_category_key == category_key, ) ).first() try: payload = adapter.discover_taxonomy( category_key, source_url=None if existing is None else existing.source_url, ) except TypeError: payload = adapter.discover_taxonomy(category_key) except AppError as exc: category = self.session.scalars( select(SourceCategoryRecord).where( SourceCategoryRecord.source_id == source_id, SourceCategoryRecord.external_category_key == category_key, ) ).first() if category is not None and exc.code == "SOURCE_BLOCKED": category.taxonomy_status = TaxonomyStatus.BLOCKED.value category.last_taxonomy_scan_at = now_utc() self.session.flush() raise now = now_utc() category = self.session.scalars( select(SourceCategoryRecord).where( SourceCategoryRecord.source_id == source_id, SourceCategoryRecord.external_category_key == payload.get("external_category_key", category_key), ) ).first() if category is None: category = SourceCategoryRecord( id=uuid4(), source_id=source_id, external_category_key=str(payload.get("external_category_key") or category_key), name=str(payload.get("name") or category_key), source_url=str(payload.get("source_url") or ""), canonical_category=payload.get("canonical_category"), product_category_id=_linked_product_category(self.session, payload.get("canonical_category")), status="ACTIVE", taxonomy_status=TaxonomyStatus.LEARNING.value, last_taxonomy_scan_at=now, last_verified_at=now, ) self.session.add(category) self.session.flush() else: category.name = str(payload.get("name") or category.name) category.source_url = str(payload.get("source_url") or category.source_url) if payload.get("canonical_category"): category.canonical_category = payload.get("canonical_category") category.product_category_id = _linked_product_category(self.session, payload.get("canonical_category")) category.taxonomy_status = TaxonomyStatus.LEARNING.value category.last_taxonomy_scan_at = now category.last_verified_at = now for raw in payload.get("filters") or []: filt = self.session.scalars( select(SourceFilterDefinitionRecord).where( SourceFilterDefinitionRecord.source_category_id == category.id, SourceFilterDefinitionRecord.external_filter_key == str(raw["external_filter_key"]), ) ).first() filter_type = str(raw.get("filter_type") or "UNKNOWN") if filt is None: filt = SourceFilterDefinitionRecord( id=uuid4(), source_category_id=category.id, external_filter_key=str(raw["external_filter_key"]), label=str(raw.get("label") or raw["external_filter_key"]), group_label=raw.get("group_label") or "Allgemein", filter_type=filter_type, canonical_unit=raw.get("canonical_unit"), status=MappingStatus.LEARNED_UNVERIFIED.value, ) self.session.add(filt) self.session.flush() else: if filt.filter_type != filter_type: self._break_mappings(category.id, filt.external_filter_key) filt.label = str(raw.get("label") or filt.label) filt.group_label = raw.get("group_label") or filt.group_label or "Allgemein" filt.filter_type = filter_type filt.canonical_unit = raw.get("canonical_unit") self._sync_options(filt, raw.get("options") or []) if raw.get("range"): self._sync_range(filt, raw["range"]) self._sync_learned_mapping(category, filt, raw) requested = str(payload.get("taxonomy_status") or "") if requested in {item.value for item in TaxonomyStatus}: category.taxonomy_status = requested else: category.taxonomy_status = ( TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value ) self.session.flush() return self.filters(source_id, category.external_category_key) def taxonomy_status(self, source_id: str, category_key: str) -> dict[str, Any]: category = self._category(source_id, category_key) filters = self.session.scalars( select(SourceFilterDefinitionRecord).where(SourceFilterDefinitionRecord.source_category_id == category.id) ).all() mappings = self.session.scalars( select(CriterionMappingRecord).where(CriterionMappingRecord.source_category_id == category.id) ).all() return jsonable( { "source_category_status": category.taxonomy_status, "filters_discovered": len(filters), "filters_verified": sum(1 for row in mappings if row.status == MappingStatus.VERIFIED.value), "filters_unverified": sum( 1 for row in mappings if row.status == MappingStatus.LEARNED_UNVERIFIED.value ), "filters_broken": sum(1 for row in mappings if row.status == MappingStatus.BROKEN.value), "last_scan_at": category.last_taxonomy_scan_at, } ) def filters(self, source_id: str, category_key: str) -> dict[str, Any]: category = self._category(source_id, category_key) rows = self.session.scalars( select(SourceFilterDefinitionRecord) .where(SourceFilterDefinitionRecord.source_category_id == category.id) .order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label) ).all() filters = [] for filt in rows: options = self.session.scalars( select(SourceFilterOptionRecord) .where(SourceFilterOptionRecord.filter_id == filt.id) .order_by(SourceFilterOptionRecord.source_value) ).all() rng = self.session.get(SourceFilterRangeRecord, filt.id) filters.append( { "id": filt.id, "external_filter_key": filt.external_filter_key, "label": filt.label, "group_label": filt.group_label or "Allgemein", "filter_type": filt.filter_type, "canonical_unit": filt.canonical_unit, "status": filt.status, "options": [ { "source_value": option.source_value, "normalized_value": option.normalized_value, "operator": option.operator, "unit": option.unit, } for option in options ], "range": None if rng is None else { "minimum": rng.minimum, "maximum": rng.maximum, "step": rng.step, "unit": rng.unit, }, } ) return jsonable( { "id": category.id, "source_id": category.source_id, "external_category_key": category.external_category_key, "name": category.name, "source_url": category.source_url, "canonical_category": category.canonical_category, "product_category_id": category.product_category_id, "status": category.status, "taxonomy_status": category.taxonomy_status, "last_verified_at": category.last_verified_at, "filters": filters, } ) def _sync_options(self, filt: SourceFilterDefinitionRecord, options: list[dict[str, Any]]) -> None: seen: set[str] = set() existing = { row.source_value: row for row in self.session.scalars( select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id) ) } for raw in options: value = str(raw["source_value"]) seen.add(value) row = existing.get(value) if row is None: self.session.add( SourceFilterOptionRecord( id=uuid4(), filter_id=filt.id, source_value=value, normalized_value=raw.get("normalized_value"), operator=raw.get("operator"), unit=raw.get("unit"), ) ) else: row.normalized_value = raw.get("normalized_value") row.operator = raw.get("operator") row.unit = raw.get("unit") removed = set(existing) - seen if removed: self._break_mappings(filt.source_category_id, filt.external_filter_key) def _sync_learned_mapping( self, category: SourceCategoryRecord, filt: SourceFilterDefinitionRecord, raw: dict[str, Any], ) -> None: encoding = raw.get("encoding") or {"url_param": "xf"} definition = upsert_definition( self.session, { "key": f"{category.source_id}:{category.external_category_key}:{filt.external_filter_key}", "origin": "SOURCE", "source": category.source_id, "product_category_id": category.product_category_id, "label": filt.label, "value_type": "number" if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value} else "string", "unit": filt.canonical_unit, "category": category.product_category_id, }, ) mapping = self.session.scalars( select(CriterionMappingRecord).where( CriterionMappingRecord.source_category_id == category.id, CriterionMappingRecord.external_filter_key == filt.external_filter_key, ) ).first() options = raw.get("options") or [] source_value = None if options: source_value = str(options[0].get("query_value") or options[0].get("source_value")) now = now_utc() if mapping is None: mapping = CriterionMappingRecord( id=uuid4(), criterion_definition_id=definition.id, criterion_key=definition.key, source_id=category.source_id, source_category_id=category.id, external_filter_key=filt.external_filter_key, source_value=source_value, encoding=encoding, status=MappingStatus.LEARNED_UNVERIFIED.value, definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})), created_at=now, updated_at=now, ) self.session.add(mapping) self.session.flush() else: mapping.criterion_definition_id = definition.id mapping.criterion_key = definition.key mapping.source_value = source_value mapping.encoding = encoding mapping.definition_hash = sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})) mapping.updated_at = now probe_value = source_value or 0 if options and options[0].get("normalized_value") is not None: probe_value = options[0]["normalized_value"] operator = (options[0].get("operator") if options else None) or "<=" if filt.filter_type == FilterType.NUMERIC_RANGE.value: operator = ">=" probe_value = 1000 verified = verify_roundtrip( category_url=category.source_url or f"https://geizhals.de/?cat={category.external_category_key}", criterion={ "criterion": definition.key, "operator": operator, "value": probe_value, }, mapping={ "criterion_key": definition.key, "external_filter_key": filt.external_filter_key, "source_value": source_value, "encoding": encoding, }, ) if mapping.status != MappingStatus.DISABLED.value and verified: mapping.status = MappingStatus.VERIFIED.value elif mapping.status == MappingStatus.BROKEN.value: mapping.status = MappingStatus.LEARNED_UNVERIFIED.value def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None: row = self.session.get(SourceFilterRangeRecord, filt.id) if row is None: self.session.add( SourceFilterRangeRecord( filter_id=filt.id, minimum=raw.get("minimum"), maximum=raw.get("maximum"), step=raw.get("step"), unit=raw.get("unit"), ) ) return row.minimum = raw.get("minimum") row.maximum = raw.get("maximum") row.step = raw.get("step") row.unit = raw.get("unit") def _break_mappings(self, category_id: UUID, filter_key: str) -> None: now = now_utc() for mapping in self.session.scalars( select(CriterionMappingRecord).where( CriterionMappingRecord.source_category_id == category_id, CriterionMappingRecord.external_filter_key == filter_key, CriterionMappingRecord.status == MappingStatus.VERIFIED.value, ) ): mapping.status = MappingStatus.BROKEN.value mapping.updated_at = now self._mark_searches_out_of_sync(category_id) def _mark_searches_out_of_sync(self, category_id: UUID) -> None: now = now_utc() for search in self.session.scalars( select(DiscoverySearchRecord).where(DiscoverySearchRecord.source_category_id == category_id) ): if search.status not in {SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}: search.status = SearchStatus.OUT_OF_SYNC.value search.updated_at = now def _category(self, source_id: str, category_key: str) -> SourceCategoryRecord: category = self.session.scalars( select(SourceCategoryRecord).where( SourceCategoryRecord.source_id == source_id, SourceCategoryRecord.external_category_key == category_key, ) ).first() if category is None: raise AppError(404, "NOT_FOUND", "source category not found") return category class CriterionDefinitionService: def __init__(self, session: Session) -> None: self.session = session def create(self, payload: dict[str, Any]) -> dict[str, Any]: key = str(payload["key"]) if get_definition_by_key(self.session, key) is not None: raise AppError(409, "CONFLICT", "criterion definition already exists", {"key": key}) row = upsert_definition(self.session, payload) return self._out(row) def list(self, category: str | None = None) -> list[dict[str, Any]]: stmt = select(CriterionDefinitionRecord).order_by(CriterionDefinitionRecord.key) if category is not None: stmt = stmt.where( (CriterionDefinitionRecord.category == category) | (CriterionDefinitionRecord.product_category_id == category) | (CriterionDefinitionRecord.category.is_(None)) ) return [self._out(row) for row in self.session.scalars(stmt)] def _out(self, row: CriterionDefinitionRecord) -> dict[str, Any]: return jsonable( { "id": row.id, "key": row.key, "origin": row.origin, "source": row.source, "product_category_id": row.product_category_id, "label": row.label, "value_type": row.value_type, "unit": row.unit, "semantic": row.semantic, "category": row.category, "description": row.description, "scoring_attribute_key": row.scoring_attribute_key, } ) class CriterionMappingService: def __init__(self, session: Session) -> None: self.session = session def create(self, payload: dict[str, Any]) -> dict[str, Any]: category = self.session.get(SourceCategoryRecord, payload["source_category_id"]) if category is None: raise AppError(404, "NOT_FOUND", "source category not found") definition = self._ensure_definition(payload) filt = self.session.scalars( select(SourceFilterDefinitionRecord).where( SourceFilterDefinitionRecord.source_category_id == category.id, SourceFilterDefinitionRecord.external_filter_key == str(payload["external_filter_key"]), ) ).first() if filt is None: raise AppError(422, "UNKNOWN_FILTER", "filter is not in the learned taxonomy") now = now_utc() encoding = payload.get("encoding") or {"url_param": "xf"} mapping = CriterionMappingRecord( id=uuid4(), criterion_definition_id=definition.id, criterion_key=definition.key, source_id=category.source_id, source_category_id=category.id, external_filter_key=str(payload["external_filter_key"]), source_value=payload.get("source_value"), encoding=encoding, status=str(payload.get("status") or MappingStatus.LEARNED_UNVERIFIED.value), definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": payload.get("source_value")})), created_at=now, updated_at=now, ) if mapping.status != MappingStatus.DISABLED.value and self._auto_verify(category, mapping, payload): mapping.status = MappingStatus.VERIFIED.value self.session.add(mapping) self.session.flush() self._mark_out_of_sync(category.id) return self._out(mapping) def patch(self, mapping_id: UUID, payload: dict[str, Any]) -> dict[str, Any]: mapping = self.session.get(CriterionMappingRecord, mapping_id) if mapping is None: raise AppError(404, "NOT_FOUND", "criterion mapping not found") if "source_value" in payload: mapping.source_value = payload["source_value"] if "encoding" in payload: mapping.encoding = payload["encoding"] if "status" in payload: mapping.status = str(payload["status"]) mapping.definition_hash = sha256_text( canonical_json({"encoding": mapping.encoding, "source_value": mapping.source_value}) ) mapping.updated_at = now_utc() category = self.session.get(SourceCategoryRecord, mapping.source_category_id) if category is not None and mapping.status != MappingStatus.DISABLED.value: if self._auto_verify(category, mapping, {"probe_value": payload.get("probe_value")}): mapping.status = MappingStatus.VERIFIED.value self.session.flush() if category is not None: self._mark_out_of_sync(category.id) return self._out(mapping) def list(self, source_category_id: UUID | None = None) -> list[dict[str, Any]]: stmt = select(CriterionMappingRecord).order_by(CriterionMappingRecord.created_at) if source_category_id is not None: stmt = stmt.where(CriterionMappingRecord.source_category_id == source_category_id) return [self._out(row) for row in self.session.scalars(stmt)] def verified_for(self, category_id: UUID) -> list[CriterionMappingRecord]: return list( self.session.scalars( select(CriterionMappingRecord).where( CriterionMappingRecord.source_category_id == category_id, CriterionMappingRecord.status == MappingStatus.VERIFIED.value, ) ) ) def _ensure_definition(self, payload: dict[str, Any]) -> CriterionDefinitionRecord: if payload.get("criterion_definition_id"): existing = get_definition(self.session, payload["criterion_definition_id"]) if existing is None: raise AppError(422, "UNKNOWN_CRITERION", "criterion definition not found") return existing key = payload.get("criterion_key") if not key: raise AppError(422, "UNKNOWN_CRITERION", "criterion_definition_id or criterion_key is required") existing = get_definition_by_key(self.session, str(key)) if existing is not None: return existing value_type = payload.get("value_type") if not value_type: raise AppError(422, "UNKNOWN_CRITERION", "criterion definition missing; provide value_type") return upsert_definition( self.session, { "key": str(key), "origin": "SOURCE", "source": payload.get("source_id"), "label": str(key), "value_type": str(value_type), "unit": payload.get("unit"), "semantic": payload.get("semantic"), "category": payload.get("category"), "description": payload.get("description"), "scoring_attribute_key": str(key), }, ) def _auto_verify( self, category: SourceCategoryRecord, mapping: CriterionMappingRecord, payload: dict[str, Any], ) -> bool: criterion = { "criterion": mapping.criterion_key, "operator": payload.get("operator") or ">=", "value": payload.get("probe_value") if payload.get("probe_value") is not None else mapping.source_value or 0, } return verify_roundtrip( category_url=category.source_url, criterion=criterion, mapping={ "criterion_key": mapping.criterion_key, "external_filter_key": mapping.external_filter_key, "source_value": mapping.source_value, "encoding": mapping.encoding, }, ) def _mark_out_of_sync(self, category_id: UUID) -> None: now = now_utc() current = mapping_version_hash(self.verified_for(category_id)) for search in self.session.scalars( select(DiscoverySearchRecord).where(DiscoverySearchRecord.source_category_id == category_id) ): if search.mapping_version != current and search.status not in { SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value, }: search.status = SearchStatus.OUT_OF_SYNC.value search.updated_at = now def _out(self, row: CriterionMappingRecord) -> dict[str, Any]: return jsonable( { "id": row.id, "criterion_definition_id": row.criterion_definition_id, "criterion_key": row.criterion_key, "source_id": row.source_id, "source_category_id": row.source_category_id, "external_filter_key": row.external_filter_key, "source_value": row.source_value, "encoding": row.encoding, "status": row.status, "definition_hash": row.definition_hash, "created_at": row.created_at, "updated_at": row.updated_at, } ) class DiscoverySearchService: def __init__(self, session: Session, adapter: DiscoverySourceAdapter | None = None) -> None: self.session = session self.adapter = adapter self.projects = ProjectService(session) self.mappings = CriterionMappingService(session) def create(self, payload: dict[str, Any]) -> dict[str, Any]: project = self.session.get(BuyingProjectRecord, payload["buying_project_id"]) if project is None: raise AppError(404, "NOT_FOUND", "buying project not found") category = self.session.get(SourceCategoryRecord, payload["source_category_id"]) if category is None: raise AppError(404, "NOT_FOUND", "source category not found") criteria = self.projects.latest_criteria(project.id) if criteria is None: raise AppError(422, "CRITERIA_REQUIRED", "buying project has no criteria") now = now_utc() interval = int(payload.get("schedule_interval_minutes") or 180) if interval < 1: raise AppError(422, "VALIDATION_ERROR", "schedule_interval_minutes must be > 0") existing_active = self.session.scalars( select(DiscoverySearchRecord).where( DiscoverySearchRecord.buying_project_id == project.id, DiscoverySearchRecord.source_id == category.source_id, DiscoverySearchRecord.source_category_id == category.id, DiscoverySearchRecord.status.in_( (SearchStatus.ACTIVE.value, SearchStatus.OUT_OF_SYNC.value, SearchStatus.TOO_BROAD.value) ), ) ).first() if existing_active is not None: raise AppError(409, "SEARCH_EXISTS", "an active discovery search already exists for this source") verified = self.mappings.verified_for(category.id) url, _internal = generate_search_url( category_url=category.source_url, criteria_items=self.projects.criteria_items(project.id, criteria.version), mappings=[self._mapping_dict(row) for row in verified], ) search = DiscoverySearchRecord( id=uuid4(), buying_project_id=project.id, source_id=category.source_id, source_category_id=category.id, criteria_version=criteria.version, mapping_version=mapping_version_hash(verified), generated_url=url, status=SearchStatus.ACTIVE.value, schedule_interval_minutes=interval, broad_result_threshold=int(payload.get("broad_result_threshold") or 300), next_run_at=now, last_successful_run_at=None, created_at=now, updated_at=now, ) self.session.add(search) self.session.flush() return self.get(search.id) def generate_query(self, search_id: UUID) -> dict[str, Any]: search = self._require_search(search_id) project = self.session.get(BuyingProjectRecord, search.buying_project_id) criteria = self.projects.latest_criteria(search.buying_project_id) category = self.session.get(SourceCategoryRecord, search.source_category_id) if project is None or criteria is None or category is None: raise AppError(404, "NOT_FOUND", "search dependencies missing") verified = self.mappings.verified_for(category.id) url, internal_only = generate_search_url( category_url=category.source_url, criteria_items=self.projects.criteria_items(search.buying_project_id, criteria.version), mappings=[self._mapping_dict(row) for row in verified], ) search.criteria_version = criteria.version search.mapping_version = mapping_version_hash(verified) search.generated_url = url if search.status == SearchStatus.OUT_OF_SYNC.value: search.status = SearchStatus.ACTIVE.value search.next_run_at = now_utc() search.updated_at = now_utc() self.session.flush() body = self.get(search.id) body["internal_only"] = internal_only return body def get(self, search_id: UUID) -> dict[str, Any]: search = self._require_search(search_id) return jsonable( { "id": search.id, "buying_project_id": search.buying_project_id, "source_id": search.source_id, "source_category_id": search.source_category_id, "criteria_version": search.criteria_version, "mapping_version": search.mapping_version, "generated_url": search.generated_url, "status": search.status, "schedule_interval_minutes": search.schedule_interval_minutes, "broad_result_threshold": search.broad_result_threshold, "next_run_at": search.next_run_at, "last_successful_run_at": search.last_successful_run_at, "created_at": search.created_at, "updated_at": search.updated_at, } ) def list_for_project(self, project_id: UUID) -> list[dict[str, Any]]: if self.session.get(BuyingProjectRecord, project_id) is None: raise AppError(404, "NOT_FOUND", "buying project not found") rows = self.session.scalars( select(DiscoverySearchRecord) .where(DiscoverySearchRecord.buying_project_id == project_id) .order_by(DiscoverySearchRecord.created_at) ).all() return [self.get(row.id) for row in rows] def patch(self, search_id: UUID, payload: dict[str, Any]) -> dict[str, Any]: search = self._require_search(search_id) if "schedule_interval_minutes" in payload: interval = int(payload["schedule_interval_minutes"]) if interval < 1: raise AppError(422, "VALIDATION_ERROR", "schedule_interval_minutes must be > 0") search.schedule_interval_minutes = interval if "status" in payload: status = str(payload["status"]) if status not in {SearchStatus.ACTIVE.value, SearchStatus.DISABLED.value}: raise AppError(422, "VALIDATION_ERROR", "status may only be ACTIVE or DISABLED") search.status = status search.updated_at = now_utc() self.session.flush() return self.get(search.id) def list_all(self) -> list[dict[str, Any]]: rows = self.session.scalars(select(DiscoverySearchRecord).order_by(DiscoverySearchRecord.created_at)).all() return [self.get(row.id) for row in rows] def list_due(self) -> list[dict[str, Any]]: now = now_utc() rows = self.session.scalars( select(DiscoverySearchRecord) .join(BuyingProjectRecord, BuyingProjectRecord.id == DiscoverySearchRecord.buying_project_id) .where( DiscoverySearchRecord.next_run_at <= now, DiscoverySearchRecord.status.in_((SearchStatus.ACTIVE.value, SearchStatus.TOO_BROAD.value)), BuyingProjectRecord.status == ProjectStatus.ACTIVE.value, ) .order_by(DiscoverySearchRecord.next_run_at) ).all() return [self.get(row.id) for row in rows] def list_runs(self, search_id: UUID) -> list[dict[str, Any]]: self._require_search(search_id) rows = self.session.scalars( select(SearchRunRecord) .where(SearchRunRecord.discovery_search_id == search_id) .order_by(SearchRunRecord.started_at.desc()) ).all() return [self._run_out(row) for row in rows] def get_run(self, run_id: UUID) -> dict[str, Any]: run = self.session.get(SearchRunRecord, run_id) if run is None: raise AppError(404, "NOT_FOUND", "search run not found") items = self.session.scalars(select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == run.id)).all() body = self._run_out(run) body["items"] = [ jsonable( { "id": item.id, "source_product_reference_id": item.source_product_reference_id, "observation_id": item.observation_id, "diff_status": item.diff_status, } ) for item in items ] return body def run(self, search_id: UUID) -> dict[str, Any]: return SearchExecutionService(self.session, self.adapter).execute(search_id) def _require_search(self, search_id: UUID) -> DiscoverySearchRecord: search = self.session.get(DiscoverySearchRecord, search_id) if search is None: raise AppError(404, "NOT_FOUND", "discovery search not found") return search def _mapping_dict(self, row: CriterionMappingRecord) -> dict[str, Any]: return { "criterion_key": row.criterion_key, "external_filter_key": row.external_filter_key, "source_value": row.source_value, "encoding": row.encoding, "status": row.status, } def _run_out(self, run: SearchRunRecord) -> dict[str, Any]: return jsonable( { "id": run.id, "discovery_search_id": run.discovery_search_id, "criteria_version": run.criteria_version, "mapping_version": run.mapping_version, "generated_url": run.generated_url, "started_at": run.started_at, "finished_at": run.finished_at, "status": run.status, "result_count": run.result_count, "page_count": run.page_count, "new_count": run.new_count, "changed_count": run.changed_count, "unchanged_count": run.unchanged_count, "missing_count": run.missing_count, "http_status_summary": run.http_status_summary, "error_code": run.error_code, } ) class SearchExecutionService: def __init__(self, session: Session, adapter: DiscoverySourceAdapter | None = None) -> None: self.session = session self.adapter = adapter self.searches = DiscoverySearchService(session, adapter) self.projects = ProjectService(session) self.mappings = CriterionMappingService(session) self.catalog = CatalogRepository(session) def execute(self, search_id: UUID) -> dict[str, Any]: search = self.session.get(DiscoverySearchRecord, search_id) if search is None: raise AppError(404, "NOT_FOUND", "discovery search not found") project = self.session.get(BuyingProjectRecord, search.buying_project_id) if project is None or project.status != ProjectStatus.ACTIVE.value: raise AppError(409, "PROJECT_INACTIVE", "buying project is not active") if search.status == SearchStatus.OUT_OF_SYNC.value: raise AppError(409, "SEARCH_OUT_OF_SYNC", "generate-query is required before running") if search.status in {SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}: raise AppError(409, "SEARCH_DISABLED", "discovery search is disabled") criteria = self.projects.criteria_version(project.id, search.criteria_version) category = self.session.get(SourceCategoryRecord, search.source_category_id) if criteria is None or category is None: raise AppError(422, "SEARCH_INVALID", "search is missing criteria or category") now = now_utc() run = SearchRunRecord( id=uuid4(), discovery_search_id=search.id, criteria_version=search.criteria_version, mapping_version=search.mapping_version, generated_url=search.generated_url, started_at=now, finished_at=None, status=RunStatus.RUNNING.value, result_count=0, page_count=0, new_count=0, changed_count=0, unchanged_count=0, missing_count=0, http_status_summary={"pages": []}, error_code=None, ) self.session.add(run) self.session.flush() adapter = self.adapter or discovery_adapter(search.source_id) pages: list[dict[str, Any]] = [] items: list[dict[str, Any]] = [] try: pages, items, error_code, advertised = self._paginate(adapter, search) except AppError as exc: run.status = RunStatus.FAILED.value run.error_code = exc.code run.finished_at = now_utc() search.updated_at = run.finished_at self.session.flush() return self.searches.get_run(run.id) run.page_count = len(pages) run.result_count = max(len(items), advertised) run.http_status_summary = {"pages": pages} run.error_code = error_code run.finished_at = now_utc() if error_code == "BLOCKED": run.status = RunStatus.BLOCKED.value search.status = SearchStatus.BLOCKED.value search.next_run_at = None elif error_code == "BACKOFF": run.status = RunStatus.PARTIAL.value search.next_run_at = run.finished_at + self._backoff(search.id) elif error_code == "PARTIAL": run.status = RunStatus.PARTIAL.value search.next_run_at = run.finished_at + timedelta(minutes=search.schedule_interval_minutes) else: run.status = RunStatus.SUCCESS.value search.last_successful_run_at = run.finished_at search.next_run_at = run.finished_at + timedelta(minutes=search.schedule_interval_minutes) if run.result_count > search.broad_result_threshold: run.status = RunStatus.TOO_BROAD.value search.status = SearchStatus.TOO_BROAD.value elif search.status == SearchStatus.TOO_BROAD.value: search.status = SearchStatus.ACTIVE.value search.updated_at = run.finished_at self.session.flush() if run.status in {RunStatus.SUCCESS.value, RunStatus.PARTIAL.value, RunStatus.TOO_BROAD.value}: self._persist_results(search, run, category, items) self._apply_run_counts(run) CandidateRankingService(self.session).refresh(project.id) self.session.flush() return self.searches.get_run(run.id) def _paginate( self, adapter: DiscoverySourceAdapter, search: DiscoverySearchRecord, ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None, int]: query = {"url": search.generated_url} collected: list[dict[str, Any]] = [] pages: list[dict[str, Any]] = [] page = 1 expected_pages = 1 advertised = 0 while page <= expected_pages and page <= MAX_PAGES: try: response = adapter.fetch_results(query, {"page": page}) except AppError as exc: if exc.code == "FIXTURE_NOT_FOUND" and page > 1: return pages, collected, "PARTIAL", advertised raise status = int(response.get("http_status") or 200) pages.append({"page": page, "status": status, "result_count": response.get("result_count")}) if status == 429: return pages, collected, "BACKOFF", advertised if status == 403 or response.get("challenge") == "captcha" or response.get("blocked"): return pages, collected, "BLOCKED", advertised if status >= 400: return pages, collected, "PARTIAL", advertised collected.extend(adapter.parse_results(response)) if response.get("result_count") is not None: advertised = max(advertised, int(response["result_count"])) if advertised > search.broad_result_threshold: return pages, collected, None, advertised expected_pages = int(response.get("page_count") or page) page += 1 if page > MAX_PAGES + 1 or (expected_pages > MAX_PAGES and len(pages) >= MAX_PAGES): return pages, collected, "PARTIAL", advertised return pages, collected, None, advertised def _backoff(self, search_id: UUID) -> timedelta: rows = self.session.scalars( select(SearchRunRecord) .where(SearchRunRecord.discovery_search_id == search_id) .order_by(SearchRunRecord.started_at.desc()) .limit(8) ).all() streak = 0 for row in rows: if row.error_code == "BACKOFF": streak += 1 continue break return timedelta(minutes=BACKOFF_BASE_MINUTES * (2 ** min(streak, 5))) def _persist_results( self, search: DiscoverySearchRecord, run: SearchRunRecord, category: SourceCategoryRecord, items: list[dict[str, Any]], ) -> None: adapter = self.adapter or discovery_adapter(search.source_id) known_rows = list( self.session.scalars( select(SourceProductReferenceRecord).where( SourceProductReferenceRecord.source_id == search.source_id, SourceProductReferenceRecord.source_category_id == category.id, ) ) ) known = [(row.id, self._signals_from_ref(row)) for row in known_rows] current_ids: set[UUID] = set() current_obs: dict[UUID, dict[str, Any]] = {} observations: dict[UUID, ProductObservationRecord] = {} for raw in items: normalized = adapter.normalize_result(raw) incoming = self._signals_from_norm(normalized) decision = decide_identity(incoming, known) if decision.match_status == MatchStatus.MATCHED and decision.existing_id is not None: ref = self.session.get(SourceProductReferenceRecord, decision.existing_id) assert ref is not None self._touch_ref(ref, normalized, incoming, MatchStatus.MATCHED) else: ref = self._new_ref(search, category, normalized, incoming, decision.match_status) known.append((ref.id, incoming)) known_rows.append(ref) current_ids.add(ref.id) obs = ProductObservationRecord( id=uuid4(), source_product_reference_id=ref.id, search_run_id=run.id, observed_at=run.finished_at or now_utc(), raw_title=str(normalized.get("source_title") or ""), best_price=None if normalized.get("best_price") is None else Decimal(str(normalized["best_price"])), currency=normalized.get("currency"), availability=normalized.get("availability"), seller_count=normalized.get("seller_count"), ) self.session.add(obs) self.session.flush() _store_observation_attributes( self.session, obs.id, normalized.get("raw_attributes") or {}, normalized.get("normalized_attributes") or {}, ) observations[ref.id] = obs current_obs[ref.id] = _obs_dict(self.session, obs) self._upsert_candidate(search.buying_project_id, ref, decision.match_status) self.session.flush() previous_ids, previous_obs = self._previous_success(search.id, run.id) diffs = diff_run( previous_ids=previous_ids, current_ids=current_ids, previous_obs=previous_obs, current_obs=current_obs, run_status=run.status, ) for ref_id, status in diffs.items(): obs = observations.get(ref_id) self.session.add( SearchRunItemRecord( id=uuid4(), search_run_id=run.id, source_product_reference_id=ref_id, observation_id=None if obs is None else obs.id, diff_status=status.value, ) ) ref = self.session.get(SourceProductReferenceRecord, ref_id) if ref is None: continue missing = run.status == RunStatus.SUCCESS.value and status == DiffStatus.MISSING reappeared = ref_id in current_ids and ref.lifecycle_status != LifecycleStatus.ACTIVE.value lifecycle, streak = next_lifecycle( current=ref.lifecycle_status, missing_in_success_run=missing, consecutive_missing=ref.consecutive_missing_success_runs, reappeared=reappeared or (ref_id in current_ids and status != DiffStatus.MISSING), ) if ref.lifecycle_status == LifecycleStatus.ARCHIVED.value: continue if ref_id in current_ids: ref.lifecycle_status = LifecycleStatus.ACTIVE.value ref.consecutive_missing_success_runs = 0 else: ref.lifecycle_status = lifecycle ref.consecutive_missing_success_runs = streak def _apply_run_counts(self, run: SearchRunRecord) -> None: items = self.session.scalars( select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == run.id) ).all() run.new_count = sum(1 for item in items if item.diff_status == DiffStatus.NEW.value) run.changed_count = sum(1 for item in items if item.diff_status == DiffStatus.CHANGED.value) run.unchanged_count = sum(1 for item in items if item.diff_status == DiffStatus.UNCHANGED.value) run.missing_count = sum(1 for item in items if item.diff_status == DiffStatus.MISSING.value) def _previous_success(self, search_id: UUID, current_run_id: UUID) -> tuple[set[UUID], dict[UUID, dict[str, Any]]]: runs = self.session.scalars( select(SearchRunRecord) .where( SearchRunRecord.discovery_search_id == search_id, SearchRunRecord.status == RunStatus.SUCCESS.value, SearchRunRecord.id != current_run_id, ) .order_by(SearchRunRecord.started_at.asc()) ).all() known: set[UUID] = set() latest_obs: dict[UUID, dict[str, Any]] = {} for run in runs: items = self.session.scalars( select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == run.id) ).all() known.update( item.source_product_reference_id for item in items if item.diff_status != DiffStatus.MISSING.value ) for row in self.session.scalars( select(ProductObservationRecord).where(ProductObservationRecord.search_run_id == run.id) ): latest_obs[row.source_product_reference_id] = _obs_dict(self.session, row) return known, latest_obs def _new_ref( self, search: DiscoverySearchRecord, category: SourceCategoryRecord, normalized: dict[str, Any], incoming: IdentitySignals, match_status: MatchStatus, ) -> SourceProductReferenceRecord: now = now_utc() ref = SourceProductReferenceRecord( id=uuid4(), source_id=search.source_id, source_category_id=category.id, source_external_id=normalized.get("source_external_id"), source_url=normalized.get("source_url"), source_title=str(normalized.get("source_title") or ""), manufacturer=normalized.get("manufacturer"), model=normalized.get("model"), manufacturer_sku=normalized.get("manufacturer_sku"), gtin_ean=normalized.get("gtin_ean"), normalized_spec_fingerprint=incoming.fingerprint, first_seen_at=now, last_seen_at=now, lifecycle_status=LifecycleStatus.ACTIVE.value, match_status=match_status.value, consecutive_missing_success_runs=0, product_variant_id=None, ) self.session.add(ref) self.session.flush() return ref def _touch_ref( self, ref: SourceProductReferenceRecord, normalized: dict[str, Any], incoming: IdentitySignals, match_status: MatchStatus, ) -> None: ref.source_title = str(normalized.get("source_title") or ref.source_title) ref.source_url = normalized.get("source_url") or ref.source_url ref.manufacturer = normalized.get("manufacturer") or ref.manufacturer ref.model = normalized.get("model") or ref.model ref.manufacturer_sku = normalized.get("manufacturer_sku") or ref.manufacturer_sku ref.gtin_ean = normalized.get("gtin_ean") or ref.gtin_ean ref.normalized_spec_fingerprint = incoming.fingerprint ref.last_seen_at = now_utc() ref.match_status = match_status.value if ref.lifecycle_status != LifecycleStatus.ARCHIVED.value: ref.lifecycle_status = LifecycleStatus.ACTIVE.value ref.consecutive_missing_success_runs = 0 def _upsert_candidate(self, project_id: UUID, ref: SourceProductReferenceRecord, match_status: MatchStatus) -> None: existing = self.session.scalars( select(ProjectCandidateRecord).where( ProjectCandidateRecord.buying_project_id == project_id, ProjectCandidateRecord.source_product_reference_id == ref.id, ) ).first() now = now_utc() if existing is None: candidate = ProjectCandidateRecord( id=uuid4(), buying_project_id=project_id, source_product_reference_id=ref.id, product_variant_id=ref.product_variant_id, status=CandidateStatus.CANDIDATE.value, match_status=match_status.value, product_eligibility=None, product_score=None, created_at=now, updated_at=now, ) self.session.add(candidate) self.session.flush() self.session.add( CandidateStatusHistoryRecord( id=uuid4(), candidate_id=candidate.id, from_status=None, to_status=candidate.status, reason="discovered", created_at=now, ) ) return existing.match_status = match_status.value existing.product_variant_id = ref.product_variant_id existing.updated_at = now def _signals_from_norm(self, normalized: dict[str, Any]) -> IdentitySignals: attrs = dict(normalized.get("normalized_attributes") or {}) if normalized.get("manufacturer"): attrs.setdefault("manufacturer", normalized["manufacturer"]) if normalized.get("model"): attrs.setdefault("model", normalized["model"]) return IdentitySignals( source_external_id=normalized.get("source_external_id"), gtin_ean=normalized.get("gtin_ean"), manufacturer_sku=normalized.get("manufacturer_sku"), source_url=normalized.get("source_url"), title=str(normalized.get("source_title") or ""), manufacturer=normalized.get("manufacturer"), model=normalized.get("model"), fingerprint=spec_fingerprint(attrs), ) def _signals_from_ref(self, row: SourceProductReferenceRecord) -> IdentitySignals: return IdentitySignals( source_external_id=row.source_external_id, gtin_ean=row.gtin_ean, manufacturer_sku=row.manufacturer_sku, source_url=row.source_url, title=row.source_title, manufacturer=row.manufacturer, model=row.model, fingerprint=row.normalized_spec_fingerprint or "", ) class CandidateRankingService: def __init__(self, session: Session) -> None: self.session = session self.catalog = CatalogRepository(session) def refresh(self, project_id: UUID) -> list[dict[str, Any]]: project = self.session.get(BuyingProjectRecord, project_id) if project is None: raise AppError(404, "NOT_FOUND", "buying project not found") profile = None if project.template_id is not None and project.template_version is not None: profile = self.catalog.get_profile(project.template_id, project.template_version) model = self.catalog.get_scoring_model(profile.scoring_version) if profile is not None else None if profile is None or model is None: return self.list_candidates(project_id) definitions = self.catalog.all_definitions() candidates = self.session.scalars( select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project_id) ).all() payload: list[dict[str, Any]] = [] for candidate in candidates: ref = self.session.get(SourceProductReferenceRecord, candidate.source_product_reference_id) if ref is None or ref.lifecycle_status == LifecycleStatus.ARCHIVED.value: continue obs = self.session.scalars( select(ProductObservationRecord) .where(ProductObservationRecord.source_product_reference_id == ref.id) .order_by(ProductObservationRecord.observed_at.desc()) ).first() payload.append( { "candidate_id": candidate.id, "source_product_reference_id": ref.id, "manufacturer": None if ref is None else ref.manufacturer, "model": None if ref is None else ref.model, "source_title": None if ref is None else ref.source_title, "normalized_attributes": {} if obs is None else _attribute_map(self.session, obs.id), "lifecycle_status": ref.lifecycle_status, "match_status": candidate.match_status, "best_price": None if obs is None else obs.best_price, } ) ranked = rank_candidates( profile=profile, scoring_model=model, definitions=definitions, candidates=payload, shortlist_size=project.shortlist_size, as_of=now_utc(), ) by_id = {item["candidate_id"]: item for item in ranked} now = now_utc() for candidate in candidates: item = by_id.get(candidate.id) if item is None: continue new_status = item["candidate_status"] if candidate.status != new_status: self.session.add( CandidateStatusHistoryRecord( id=uuid4(), candidate_id=candidate.id, from_status=candidate.status, to_status=new_status, reason="ranking", created_at=now, ) ) candidate.status = new_status candidate.product_eligibility = item["product_eligibility"] candidate.product_score = item["product_score"] candidate.updated_at = now self.session.flush() return ranked def list_candidates(self, project_id: UUID, shortlisted_only: bool = False) -> list[dict[str, Any]]: project = self.session.get(BuyingProjectRecord, project_id) if project is None: raise AppError(404, "NOT_FOUND", "buying project not found") stmt = select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project_id) if shortlisted_only: stmt = stmt.where(ProjectCandidateRecord.status == CandidateStatus.SHORTLISTED.value) rows = self.session.scalars(stmt.order_by(ProjectCandidateRecord.product_score.desc().nullslast())).all() return [self._candidate_summary(project, row) for row in rows] def get_candidate(self, project_id: UUID, candidate_id: UUID) -> dict[str, Any]: project = self.session.get(BuyingProjectRecord, project_id) candidate = self.session.get(ProjectCandidateRecord, candidate_id) if project is None or candidate is None or candidate.buying_project_id != project_id: raise AppError(404, "NOT_FOUND", "candidate not found") summary = self._candidate_summary(project, candidate) ref = self.session.get(SourceProductReferenceRecord, candidate.source_product_reference_id) obs = None if ref is not None: obs = self.session.scalars( select(ProductObservationRecord) .where(ProductObservationRecord.source_product_reference_id == ref.id) .order_by(ProductObservationRecord.observed_at.desc()) ).first() summary.update( { "normalized_attributes": {} if obs is None else _attribute_map(self.session, obs.id), "source": None if ref is None else { "source_id": ref.source_id, "source_category_id": ref.source_category_id, "source_external_id": ref.source_external_id, "source_url": ref.source_url, }, "current_observation": None if obs is None else _observation_out(self.session, obs), } ) return jsonable(summary) def list_observations(self, project_id: UUID, candidate_id: UUID) -> list[dict[str, Any]]: candidate = self.session.get(ProjectCandidateRecord, candidate_id) if candidate is None or candidate.buying_project_id != project_id: raise AppError(404, "NOT_FOUND", "candidate not found") rows = self.session.scalars( select(ProductObservationRecord) .where(ProductObservationRecord.source_product_reference_id == candidate.source_product_reference_id) .order_by(ProductObservationRecord.observed_at.desc()) ).all() return [_observation_out(self.session, row) for row in rows] def _candidate_summary(self, project: BuyingProjectRecord, row: ProjectCandidateRecord) -> dict[str, Any]: ref = self.session.get(SourceProductReferenceRecord, row.source_product_reference_id) obs = None last_item = None if ref is not None: obs = self.session.scalars( select(ProductObservationRecord) .where(ProductObservationRecord.source_product_reference_id == ref.id) .order_by(ProductObservationRecord.observed_at.desc()) ).first() last_item = self.session.scalars( select(SearchRunItemRecord) .where(SearchRunItemRecord.source_product_reference_id == ref.id) .order_by(SearchRunItemRecord.id.desc()) ).first() ranking_status = ( RankingStatus.NOT_CONFIGURED.value if project.template_id is None else RankingStatus.CONFIGURED.value ) return jsonable( { "id": row.id, "candidate_id": row.id, "buying_project_id": row.buying_project_id, "source_product_reference_id": row.source_product_reference_id, "product_variant_id": row.product_variant_id, "title": None if ref is None else ref.source_title, "source_title": None if ref is None else ref.source_title, "manufacturer": None if ref is None else ref.manufacturer, "model": None if ref is None else ref.model, "source_url": None if ref is None else ref.source_url, "current_best_price": None if obs is None else obs.best_price, "currency": None if obs is None else obs.currency, "current_dynamic_attributes": [] if obs is None else _dynamic_attributes(self.session, obs.id), "product_score": row.product_score, "product_eligibility": row.product_eligibility, "ranking_status": ranking_status, "shortlist_status": row.status, "status": row.status, "current_diff_status": None if last_item is None else last_item.diff_status, "product_lifecycle_status": None if ref is None else ref.lifecycle_status, "lifecycle_status": None if ref is None else ref.lifecycle_status, "match_status": row.match_status, "first_seen_at": None if ref is None else ref.first_seen_at, "last_seen_at": None if ref is None else ref.last_seen_at, "created_at": row.created_at, "updated_at": row.updated_at, } ) def _attribute_map(session: Session, observation_id: UUID) -> dict[str, Any]: rows = session.scalars( select(ProductObservationAttributeRecord).where( ProductObservationAttributeRecord.observation_id == observation_id ) ).all() mapped: dict[str, Any] = {} for row in rows: definition = None if row.criterion_definition_id is None else session.get(CriterionDefinitionRecord, row.criterion_definition_id) key = row.attribute_key if definition is not None and definition.scoring_attribute_key: key = definition.scoring_attribute_key mapped[key] = row.normalized_value return mapped def _dynamic_attributes(session: Session, observation_id: UUID) -> list[dict[str, Any]]: rows = session.scalars( select(ProductObservationAttributeRecord).where( ProductObservationAttributeRecord.observation_id == observation_id ) ).all() return [ { "criterion_definition_id": row.criterion_definition_id, "raw_name": row.raw_name or row.attribute_key, "raw_value": row.raw_value, "normalized_value": row.normalized_value, "unit": row.unit, } for row in rows ] def _store_observation_attributes( session: Session, observation_id: UUID, raw_attributes: dict[str, Any], normalized_attributes: dict[str, Any], ) -> None: keys = set(raw_attributes) | set(normalized_attributes) definitions = { row.scoring_attribute_key or row.key: row for row in session.scalars(select(CriterionDefinitionRecord)).all() if row.scoring_attribute_key or row.key } attr_maps = { row.raw_name: row for row in session.scalars(select(SourceAttributeMappingRecord)).all() } for key in keys: mapping = attr_maps.get(key) definition = None if mapping is None else session.get(CriterionDefinitionRecord, mapping.criterion_definition_id) if definition is None: definition = definitions.get(key) session.add( ProductObservationAttributeRecord( id=uuid4(), observation_id=observation_id, criterion_definition_id=None if definition is None else definition.id, attribute_key=key, raw_name=key, raw_value=raw_attributes.get(key), normalized_value=normalized_attributes.get(key), unit=None if definition is None else definition.unit, ) ) def _obs_dict(session: Session, row: ProductObservationRecord) -> dict[str, Any]: return { "best_price": row.best_price, "availability": row.availability, "seller_count": row.seller_count, "normalized_attributes": _attribute_map(session, row.id), } def _observation_out(session: Session, row: ProductObservationRecord) -> dict[str, Any]: return jsonable( { "observation_id": row.id, "observed_at": row.observed_at, "best_price": row.best_price, "currency": row.currency, "availability": row.availability, "seller_count": row.seller_count, "dynamic_attributes": _dynamic_attributes(session, row.id), "search_run_id": row.search_run_id, } ) class SourceAttributeMappingService: def __init__(self, session: Session) -> None: self.session = session def list(self) -> list[dict[str, Any]]: rows = self.session.scalars(select(SourceAttributeMappingRecord).order_by(SourceAttributeMappingRecord.raw_name)).all() return [self._out(row) for row in rows] def create(self, payload: dict[str, Any]) -> dict[str, Any]: definition = get_definition(self.session, payload["criterion_definition_id"]) if definition is None: raise AppError(404, "NOT_FOUND", "criterion definition not found") now = now_utc() row = SourceAttributeMappingRecord( id=uuid4(), source_id=str(payload["source_id"]), source_category_id=payload.get("source_category_id"), raw_name=str(payload["raw_name"]), criterion_definition_id=definition.id, status=str(payload.get("status") or MappingStatus.LEARNED_UNVERIFIED.value), created_at=now, updated_at=now, ) self.session.add(row) self.session.flush() return self._out(row) def patch(self, mapping_id: UUID, payload: dict[str, Any]) -> dict[str, Any]: row = self.session.get(SourceAttributeMappingRecord, mapping_id) if row is None: raise AppError(404, "NOT_FOUND", "source attribute mapping not found") if "raw_name" in payload: row.raw_name = str(payload["raw_name"]) if "criterion_definition_id" in payload: row.criterion_definition_id = payload["criterion_definition_id"] if "status" in payload: row.status = str(payload["status"]) row.updated_at = now_utc() self.session.flush() return self._out(row) def _out(self, row: SourceAttributeMappingRecord) -> dict[str, Any]: return jsonable( { "id": row.id, "source_id": row.source_id, "source_category_id": row.source_category_id, "raw_name": row.raw_name, "criterion_definition_id": row.criterion_definition_id, "status": row.status, "created_at": row.created_at, "updated_at": row.updated_at, } )