1586 lines
70 KiB
Python
1586 lines
70 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
from decimal import Decimal
|
|
from typing import Any
|
|
from uuid import UUID, uuid4
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.clock import now_utc
|
|
from app.discovery.definitions import get_definition, get_definition_by_key, upsert_definition
|
|
from app.discovery.diff import diff_run, next_lifecycle
|
|
from app.discovery.enums import (
|
|
CandidateStatus,
|
|
DiffStatus,
|
|
FilterType,
|
|
LifecycleStatus,
|
|
MappingStatus,
|
|
MatchStatus,
|
|
ProjectStatus,
|
|
RankingStatus,
|
|
RunStatus,
|
|
SearchStatus,
|
|
TaxonomyStatus,
|
|
)
|
|
from app.discovery.identity import IdentitySignals, decide_identity, spec_fingerprint
|
|
from app.discovery.mapping import verify_roundtrip
|
|
from app.discovery.ranking import rank_candidates
|
|
from app.discovery.urlgen import generate_search_url
|
|
from app.domain.serialize import jsonable
|
|
from app.domain.validation import canonical_json, sha256_text
|
|
from app.errors import AppError
|
|
from app.persistence.discovery_models import (
|
|
BuyingProjectRecord,
|
|
CandidateStatusHistoryRecord,
|
|
CriterionDefinitionRecord,
|
|
CriterionMappingRecord,
|
|
DiscoverySearchRecord,
|
|
ProductCategoryRecord,
|
|
ProductObservationAttributeRecord,
|
|
ProductObservationRecord,
|
|
ProjectCandidateRecord,
|
|
SearchRunItemRecord,
|
|
SearchRunRecord,
|
|
SourceCategoryRecord,
|
|
SourceFilterDefinitionRecord,
|
|
SourceFilterOptionRecord,
|
|
SourceFilterRangeRecord,
|
|
SourceAttributeMappingRecord,
|
|
SourceProductReferenceRecord,
|
|
)
|
|
from app.persistence.repositories import CatalogRepository
|
|
from app.services.catalog_service import ensure_product_category
|
|
from app.services.project_service import ProjectService
|
|
from app.sources.base import DiscoverySourceAdapter
|
|
from app.sources.geizhals.adapter import GeizhalsAdapter
|
|
|
|
MAX_PAGES = 50
|
|
BACKOFF_BASE_MINUTES = 15
|
|
|
|
|
|
def discovery_adapter(source_id: str) -> DiscoverySourceAdapter:
|
|
if source_id == "geizhals":
|
|
return GeizhalsAdapter()
|
|
raise AppError(422, "UNKNOWN_SOURCE", "no discovery adapter for source", {"source_id": source_id})
|
|
|
|
|
|
def _linked_product_category(session: Session, canonical: str | None) -> str | None:
|
|
if not canonical:
|
|
return None
|
|
category = session.get(ProductCategoryRecord, canonical)
|
|
if category is None:
|
|
ensure_product_category(session, canonical)
|
|
return canonical
|
|
|
|
|
|
def mapping_version_hash(mappings: list[CriterionMappingRecord]) -> str:
|
|
payload = [
|
|
{
|
|
"criterion_key": row.criterion_key,
|
|
"external_filter_key": row.external_filter_key,
|
|
"source_value": row.source_value,
|
|
"encoding": row.encoding,
|
|
}
|
|
for row in mappings
|
|
if row.status == MappingStatus.VERIFIED.value
|
|
]
|
|
payload.sort(key=lambda item: (item["criterion_key"], item["external_filter_key"], str(item["source_value"])))
|
|
return sha256_text(canonical_json(payload))
|
|
|
|
|
|
class TaxonomyService:
|
|
def __init__(self, session: Session, adapter: DiscoverySourceAdapter | None = None) -> None:
|
|
self.session = session
|
|
self.adapter = adapter
|
|
|
|
def discover(self, source_id: str, category_key: str) -> dict[str, Any]:
|
|
catalog = CatalogRepository(self.session)
|
|
if catalog.get_source(source_id) is None:
|
|
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
|
adapter = self.adapter or discovery_adapter(source_id)
|
|
existing = self.session.scalars(
|
|
select(SourceCategoryRecord).where(
|
|
SourceCategoryRecord.source_id == source_id,
|
|
SourceCategoryRecord.external_category_key == category_key,
|
|
)
|
|
).first()
|
|
try:
|
|
payload = adapter.discover_taxonomy(
|
|
category_key,
|
|
source_url=None if existing is None else existing.source_url,
|
|
)
|
|
except TypeError:
|
|
payload = adapter.discover_taxonomy(category_key)
|
|
except AppError as exc:
|
|
category = self.session.scalars(
|
|
select(SourceCategoryRecord).where(
|
|
SourceCategoryRecord.source_id == source_id,
|
|
SourceCategoryRecord.external_category_key == category_key,
|
|
)
|
|
).first()
|
|
if category is not None and exc.code == "SOURCE_BLOCKED":
|
|
category.taxonomy_status = TaxonomyStatus.BLOCKED.value
|
|
category.last_taxonomy_scan_at = now_utc()
|
|
self.session.flush()
|
|
raise
|
|
now = now_utc()
|
|
category = self.session.scalars(
|
|
select(SourceCategoryRecord).where(
|
|
SourceCategoryRecord.source_id == source_id,
|
|
SourceCategoryRecord.external_category_key == payload.get("external_category_key", category_key),
|
|
)
|
|
).first()
|
|
if category is None:
|
|
category = SourceCategoryRecord(
|
|
id=uuid4(),
|
|
source_id=source_id,
|
|
external_category_key=str(payload.get("external_category_key") or category_key),
|
|
name=str(payload.get("name") or category_key),
|
|
source_url=str(payload.get("source_url") or ""),
|
|
canonical_category=payload.get("canonical_category"),
|
|
product_category_id=_linked_product_category(self.session, payload.get("canonical_category")),
|
|
status="ACTIVE",
|
|
taxonomy_status=TaxonomyStatus.LEARNING.value,
|
|
last_taxonomy_scan_at=now,
|
|
last_verified_at=now,
|
|
)
|
|
self.session.add(category)
|
|
self.session.flush()
|
|
else:
|
|
category.name = str(payload.get("name") or category.name)
|
|
category.source_url = str(payload.get("source_url") or category.source_url)
|
|
if payload.get("canonical_category"):
|
|
category.canonical_category = payload.get("canonical_category")
|
|
category.product_category_id = _linked_product_category(self.session, payload.get("canonical_category"))
|
|
category.taxonomy_status = TaxonomyStatus.LEARNING.value
|
|
category.last_taxonomy_scan_at = now
|
|
category.last_verified_at = now
|
|
for raw in payload.get("filters") or []:
|
|
filt = self.session.scalars(
|
|
select(SourceFilterDefinitionRecord).where(
|
|
SourceFilterDefinitionRecord.source_category_id == category.id,
|
|
SourceFilterDefinitionRecord.external_filter_key == str(raw["external_filter_key"]),
|
|
)
|
|
).first()
|
|
filter_type = str(raw.get("filter_type") or "UNKNOWN")
|
|
if filt is None:
|
|
filt = SourceFilterDefinitionRecord(
|
|
id=uuid4(),
|
|
source_category_id=category.id,
|
|
external_filter_key=str(raw["external_filter_key"]),
|
|
label=str(raw.get("label") or raw["external_filter_key"]),
|
|
group_label=raw.get("group_label") or "Allgemein",
|
|
filter_type=filter_type,
|
|
canonical_unit=raw.get("canonical_unit"),
|
|
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
|
)
|
|
self.session.add(filt)
|
|
self.session.flush()
|
|
else:
|
|
if filt.filter_type != filter_type:
|
|
self._break_mappings(category.id, filt.external_filter_key)
|
|
filt.label = str(raw.get("label") or filt.label)
|
|
filt.group_label = raw.get("group_label") or filt.group_label or "Allgemein"
|
|
filt.filter_type = filter_type
|
|
filt.canonical_unit = raw.get("canonical_unit")
|
|
self._sync_options(filt, raw.get("options") or [])
|
|
if raw.get("range"):
|
|
self._sync_range(filt, raw["range"])
|
|
self._sync_learned_mapping(category, filt, raw)
|
|
requested = str(payload.get("taxonomy_status") or "")
|
|
if requested in {item.value for item in TaxonomyStatus}:
|
|
category.taxonomy_status = requested
|
|
else:
|
|
category.taxonomy_status = (
|
|
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
|
|
)
|
|
self.session.flush()
|
|
return self.filters(source_id, category.external_category_key)
|
|
|
|
def taxonomy_status(self, source_id: str, category_key: str) -> dict[str, Any]:
|
|
category = self._category(source_id, category_key)
|
|
filters = self.session.scalars(
|
|
select(SourceFilterDefinitionRecord).where(SourceFilterDefinitionRecord.source_category_id == category.id)
|
|
).all()
|
|
mappings = self.session.scalars(
|
|
select(CriterionMappingRecord).where(CriterionMappingRecord.source_category_id == category.id)
|
|
).all()
|
|
return jsonable(
|
|
{
|
|
"source_category_status": category.taxonomy_status,
|
|
"filters_discovered": len(filters),
|
|
"filters_verified": sum(1 for row in mappings if row.status == MappingStatus.VERIFIED.value),
|
|
"filters_unverified": sum(
|
|
1 for row in mappings if row.status == MappingStatus.LEARNED_UNVERIFIED.value
|
|
),
|
|
"filters_broken": sum(1 for row in mappings if row.status == MappingStatus.BROKEN.value),
|
|
"last_scan_at": category.last_taxonomy_scan_at,
|
|
}
|
|
)
|
|
|
|
def filters(self, source_id: str, category_key: str) -> dict[str, Any]:
|
|
category = self._category(source_id, category_key)
|
|
rows = self.session.scalars(
|
|
select(SourceFilterDefinitionRecord)
|
|
.where(SourceFilterDefinitionRecord.source_category_id == category.id)
|
|
.order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label)
|
|
).all()
|
|
filters = []
|
|
for filt in rows:
|
|
options = self.session.scalars(
|
|
select(SourceFilterOptionRecord)
|
|
.where(SourceFilterOptionRecord.filter_id == filt.id)
|
|
.order_by(SourceFilterOptionRecord.source_value)
|
|
).all()
|
|
rng = self.session.get(SourceFilterRangeRecord, filt.id)
|
|
filters.append(
|
|
{
|
|
"id": filt.id,
|
|
"external_filter_key": filt.external_filter_key,
|
|
"label": filt.label,
|
|
"group_label": filt.group_label or "Allgemein",
|
|
"filter_type": filt.filter_type,
|
|
"canonical_unit": filt.canonical_unit,
|
|
"status": filt.status,
|
|
"options": [
|
|
{
|
|
"source_value": option.source_value,
|
|
"normalized_value": option.normalized_value,
|
|
"operator": option.operator,
|
|
"unit": option.unit,
|
|
}
|
|
for option in options
|
|
],
|
|
"range": None
|
|
if rng is None
|
|
else {
|
|
"minimum": rng.minimum,
|
|
"maximum": rng.maximum,
|
|
"step": rng.step,
|
|
"unit": rng.unit,
|
|
},
|
|
}
|
|
)
|
|
return jsonable(
|
|
{
|
|
"id": category.id,
|
|
"source_id": category.source_id,
|
|
"external_category_key": category.external_category_key,
|
|
"name": category.name,
|
|
"source_url": category.source_url,
|
|
"canonical_category": category.canonical_category,
|
|
"product_category_id": category.product_category_id,
|
|
"status": category.status,
|
|
"taxonomy_status": category.taxonomy_status,
|
|
"last_verified_at": category.last_verified_at,
|
|
"filters": filters,
|
|
}
|
|
)
|
|
|
|
def _sync_options(self, filt: SourceFilterDefinitionRecord, options: list[dict[str, Any]]) -> None:
|
|
seen: set[str] = set()
|
|
existing = {
|
|
row.source_value: row
|
|
for row in self.session.scalars(
|
|
select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id)
|
|
)
|
|
}
|
|
for raw in options:
|
|
value = str(raw["source_value"])
|
|
seen.add(value)
|
|
row = existing.get(value)
|
|
if row is None:
|
|
self.session.add(
|
|
SourceFilterOptionRecord(
|
|
id=uuid4(),
|
|
filter_id=filt.id,
|
|
source_value=value,
|
|
normalized_value=raw.get("normalized_value"),
|
|
operator=raw.get("operator"),
|
|
unit=raw.get("unit"),
|
|
)
|
|
)
|
|
else:
|
|
row.normalized_value = raw.get("normalized_value")
|
|
row.operator = raw.get("operator")
|
|
row.unit = raw.get("unit")
|
|
removed = set(existing) - seen
|
|
if removed:
|
|
self._break_mappings(filt.source_category_id, filt.external_filter_key)
|
|
|
|
def _sync_learned_mapping(
|
|
self,
|
|
category: SourceCategoryRecord,
|
|
filt: SourceFilterDefinitionRecord,
|
|
raw: dict[str, Any],
|
|
) -> None:
|
|
encoding = raw.get("encoding") or {"url_param": "xf"}
|
|
definition = upsert_definition(
|
|
self.session,
|
|
{
|
|
"key": f"{category.source_id}:{category.external_category_key}:{filt.external_filter_key}",
|
|
"origin": "SOURCE",
|
|
"source": category.source_id,
|
|
"product_category_id": category.product_category_id,
|
|
"label": filt.label,
|
|
"value_type": "number"
|
|
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
|
|
else "string",
|
|
"unit": filt.canonical_unit,
|
|
"category": category.product_category_id,
|
|
},
|
|
)
|
|
mapping = self.session.scalars(
|
|
select(CriterionMappingRecord).where(
|
|
CriterionMappingRecord.source_category_id == category.id,
|
|
CriterionMappingRecord.external_filter_key == filt.external_filter_key,
|
|
)
|
|
).first()
|
|
options = raw.get("options") or []
|
|
source_value = None
|
|
if options:
|
|
source_value = str(options[0].get("query_value") or options[0].get("source_value"))
|
|
now = now_utc()
|
|
if mapping is None:
|
|
mapping = CriterionMappingRecord(
|
|
id=uuid4(),
|
|
criterion_definition_id=definition.id,
|
|
criterion_key=definition.key,
|
|
source_id=category.source_id,
|
|
source_category_id=category.id,
|
|
external_filter_key=filt.external_filter_key,
|
|
source_value=source_value,
|
|
encoding=encoding,
|
|
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
|
definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
self.session.add(mapping)
|
|
self.session.flush()
|
|
else:
|
|
mapping.criterion_definition_id = definition.id
|
|
mapping.criterion_key = definition.key
|
|
mapping.source_value = source_value
|
|
mapping.encoding = encoding
|
|
mapping.definition_hash = sha256_text(canonical_json({"encoding": encoding, "source_value": source_value}))
|
|
mapping.updated_at = now
|
|
probe_value = source_value or 0
|
|
if options and options[0].get("normalized_value") is not None:
|
|
probe_value = options[0]["normalized_value"]
|
|
operator = (options[0].get("operator") if options else None) or "<="
|
|
if filt.filter_type == FilterType.NUMERIC_RANGE.value:
|
|
operator = ">="
|
|
probe_value = 1000
|
|
verified = verify_roundtrip(
|
|
category_url=category.source_url or f"https://geizhals.de/?cat={category.external_category_key}",
|
|
criterion={
|
|
"criterion": definition.key,
|
|
"operator": operator,
|
|
"value": probe_value,
|
|
},
|
|
mapping={
|
|
"criterion_key": definition.key,
|
|
"external_filter_key": filt.external_filter_key,
|
|
"source_value": source_value,
|
|
"encoding": encoding,
|
|
},
|
|
)
|
|
if mapping.status != MappingStatus.DISABLED.value and verified:
|
|
mapping.status = MappingStatus.VERIFIED.value
|
|
elif mapping.status == MappingStatus.BROKEN.value:
|
|
mapping.status = MappingStatus.LEARNED_UNVERIFIED.value
|
|
|
|
def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None:
|
|
row = self.session.get(SourceFilterRangeRecord, filt.id)
|
|
if row is None:
|
|
self.session.add(
|
|
SourceFilterRangeRecord(
|
|
filter_id=filt.id,
|
|
minimum=raw.get("minimum"),
|
|
maximum=raw.get("maximum"),
|
|
step=raw.get("step"),
|
|
unit=raw.get("unit"),
|
|
)
|
|
)
|
|
return
|
|
row.minimum = raw.get("minimum")
|
|
row.maximum = raw.get("maximum")
|
|
row.step = raw.get("step")
|
|
row.unit = raw.get("unit")
|
|
|
|
def _break_mappings(self, category_id: UUID, filter_key: str) -> None:
|
|
now = now_utc()
|
|
for mapping in self.session.scalars(
|
|
select(CriterionMappingRecord).where(
|
|
CriterionMappingRecord.source_category_id == category_id,
|
|
CriterionMappingRecord.external_filter_key == filter_key,
|
|
CriterionMappingRecord.status == MappingStatus.VERIFIED.value,
|
|
)
|
|
):
|
|
mapping.status = MappingStatus.BROKEN.value
|
|
mapping.updated_at = now
|
|
self._mark_searches_out_of_sync(category_id)
|
|
|
|
def _mark_searches_out_of_sync(self, category_id: UUID) -> None:
|
|
now = now_utc()
|
|
for search in self.session.scalars(
|
|
select(DiscoverySearchRecord).where(DiscoverySearchRecord.source_category_id == category_id)
|
|
):
|
|
if search.status not in {SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}:
|
|
search.status = SearchStatus.OUT_OF_SYNC.value
|
|
search.updated_at = now
|
|
|
|
def _category(self, source_id: str, category_key: str) -> SourceCategoryRecord:
|
|
category = self.session.scalars(
|
|
select(SourceCategoryRecord).where(
|
|
SourceCategoryRecord.source_id == source_id,
|
|
SourceCategoryRecord.external_category_key == category_key,
|
|
)
|
|
).first()
|
|
if category is None:
|
|
raise AppError(404, "NOT_FOUND", "source category not found")
|
|
return category
|
|
|
|
|
|
class CriterionDefinitionService:
|
|
def __init__(self, session: Session) -> None:
|
|
self.session = session
|
|
|
|
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
key = str(payload["key"])
|
|
if get_definition_by_key(self.session, key) is not None:
|
|
raise AppError(409, "CONFLICT", "criterion definition already exists", {"key": key})
|
|
row = upsert_definition(self.session, payload)
|
|
return self._out(row)
|
|
|
|
def list(self, category: str | None = None) -> list[dict[str, Any]]:
|
|
stmt = select(CriterionDefinitionRecord).order_by(CriterionDefinitionRecord.key)
|
|
if category is not None:
|
|
stmt = stmt.where(
|
|
(CriterionDefinitionRecord.category == category)
|
|
| (CriterionDefinitionRecord.product_category_id == category)
|
|
| (CriterionDefinitionRecord.category.is_(None))
|
|
)
|
|
return [self._out(row) for row in self.session.scalars(stmt)]
|
|
|
|
def _out(self, row: CriterionDefinitionRecord) -> dict[str, Any]:
|
|
return jsonable(
|
|
{
|
|
"id": row.id,
|
|
"key": row.key,
|
|
"origin": row.origin,
|
|
"source": row.source,
|
|
"product_category_id": row.product_category_id,
|
|
"label": row.label,
|
|
"value_type": row.value_type,
|
|
"unit": row.unit,
|
|
"semantic": row.semantic,
|
|
"category": row.category,
|
|
"description": row.description,
|
|
"scoring_attribute_key": row.scoring_attribute_key,
|
|
}
|
|
)
|
|
|
|
|
|
class CriterionMappingService:
|
|
def __init__(self, session: Session) -> None:
|
|
self.session = session
|
|
|
|
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
category = self.session.get(SourceCategoryRecord, payload["source_category_id"])
|
|
if category is None:
|
|
raise AppError(404, "NOT_FOUND", "source category not found")
|
|
definition = self._ensure_definition(payload)
|
|
filt = self.session.scalars(
|
|
select(SourceFilterDefinitionRecord).where(
|
|
SourceFilterDefinitionRecord.source_category_id == category.id,
|
|
SourceFilterDefinitionRecord.external_filter_key == str(payload["external_filter_key"]),
|
|
)
|
|
).first()
|
|
if filt is None:
|
|
raise AppError(422, "UNKNOWN_FILTER", "filter is not in the learned taxonomy")
|
|
now = now_utc()
|
|
encoding = payload.get("encoding") or {"url_param": "xf"}
|
|
mapping = CriterionMappingRecord(
|
|
id=uuid4(),
|
|
criterion_definition_id=definition.id,
|
|
criterion_key=definition.key,
|
|
source_id=category.source_id,
|
|
source_category_id=category.id,
|
|
external_filter_key=str(payload["external_filter_key"]),
|
|
source_value=payload.get("source_value"),
|
|
encoding=encoding,
|
|
status=str(payload.get("status") or MappingStatus.LEARNED_UNVERIFIED.value),
|
|
definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": payload.get("source_value")})),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
if mapping.status != MappingStatus.DISABLED.value and self._auto_verify(category, mapping, payload):
|
|
mapping.status = MappingStatus.VERIFIED.value
|
|
self.session.add(mapping)
|
|
self.session.flush()
|
|
self._mark_out_of_sync(category.id)
|
|
return self._out(mapping)
|
|
|
|
def patch(self, mapping_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
|
mapping = self.session.get(CriterionMappingRecord, mapping_id)
|
|
if mapping is None:
|
|
raise AppError(404, "NOT_FOUND", "criterion mapping not found")
|
|
if "source_value" in payload:
|
|
mapping.source_value = payload["source_value"]
|
|
if "encoding" in payload:
|
|
mapping.encoding = payload["encoding"]
|
|
if "status" in payload:
|
|
mapping.status = str(payload["status"])
|
|
mapping.definition_hash = sha256_text(
|
|
canonical_json({"encoding": mapping.encoding, "source_value": mapping.source_value})
|
|
)
|
|
mapping.updated_at = now_utc()
|
|
category = self.session.get(SourceCategoryRecord, mapping.source_category_id)
|
|
if category is not None and mapping.status != MappingStatus.DISABLED.value:
|
|
if self._auto_verify(category, mapping, {"probe_value": payload.get("probe_value")}):
|
|
mapping.status = MappingStatus.VERIFIED.value
|
|
self.session.flush()
|
|
if category is not None:
|
|
self._mark_out_of_sync(category.id)
|
|
return self._out(mapping)
|
|
|
|
def list(self, source_category_id: UUID | None = None) -> list[dict[str, Any]]:
|
|
stmt = select(CriterionMappingRecord).order_by(CriterionMappingRecord.created_at)
|
|
if source_category_id is not None:
|
|
stmt = stmt.where(CriterionMappingRecord.source_category_id == source_category_id)
|
|
return [self._out(row) for row in self.session.scalars(stmt)]
|
|
|
|
def verified_for(self, category_id: UUID) -> list[CriterionMappingRecord]:
|
|
return list(
|
|
self.session.scalars(
|
|
select(CriterionMappingRecord).where(
|
|
CriterionMappingRecord.source_category_id == category_id,
|
|
CriterionMappingRecord.status == MappingStatus.VERIFIED.value,
|
|
)
|
|
)
|
|
)
|
|
|
|
def _ensure_definition(self, payload: dict[str, Any]) -> CriterionDefinitionRecord:
|
|
if payload.get("criterion_definition_id"):
|
|
existing = get_definition(self.session, payload["criterion_definition_id"])
|
|
if existing is None:
|
|
raise AppError(422, "UNKNOWN_CRITERION", "criterion definition not found")
|
|
return existing
|
|
key = payload.get("criterion_key")
|
|
if not key:
|
|
raise AppError(422, "UNKNOWN_CRITERION", "criterion_definition_id or criterion_key is required")
|
|
existing = get_definition_by_key(self.session, str(key))
|
|
if existing is not None:
|
|
return existing
|
|
value_type = payload.get("value_type")
|
|
if not value_type:
|
|
raise AppError(422, "UNKNOWN_CRITERION", "criterion definition missing; provide value_type")
|
|
return upsert_definition(
|
|
self.session,
|
|
{
|
|
"key": str(key),
|
|
"origin": "SOURCE",
|
|
"source": payload.get("source_id"),
|
|
"label": str(key),
|
|
"value_type": str(value_type),
|
|
"unit": payload.get("unit"),
|
|
"semantic": payload.get("semantic"),
|
|
"category": payload.get("category"),
|
|
"description": payload.get("description"),
|
|
"scoring_attribute_key": str(key),
|
|
},
|
|
)
|
|
|
|
def _auto_verify(
|
|
self,
|
|
category: SourceCategoryRecord,
|
|
mapping: CriterionMappingRecord,
|
|
payload: dict[str, Any],
|
|
) -> bool:
|
|
criterion = {
|
|
"criterion": mapping.criterion_key,
|
|
"operator": payload.get("operator") or ">=",
|
|
"value": payload.get("probe_value") if payload.get("probe_value") is not None else mapping.source_value or 0,
|
|
}
|
|
return verify_roundtrip(
|
|
category_url=category.source_url,
|
|
criterion=criterion,
|
|
mapping={
|
|
"criterion_key": mapping.criterion_key,
|
|
"external_filter_key": mapping.external_filter_key,
|
|
"source_value": mapping.source_value,
|
|
"encoding": mapping.encoding,
|
|
},
|
|
)
|
|
|
|
def _mark_out_of_sync(self, category_id: UUID) -> None:
|
|
now = now_utc()
|
|
current = mapping_version_hash(self.verified_for(category_id))
|
|
for search in self.session.scalars(
|
|
select(DiscoverySearchRecord).where(DiscoverySearchRecord.source_category_id == category_id)
|
|
):
|
|
if search.mapping_version != current and search.status not in {
|
|
SearchStatus.BLOCKED.value,
|
|
SearchStatus.DISABLED.value,
|
|
SearchStatus.PAUSED.value,
|
|
}:
|
|
search.status = SearchStatus.OUT_OF_SYNC.value
|
|
search.updated_at = now
|
|
|
|
def _out(self, row: CriterionMappingRecord) -> dict[str, Any]:
|
|
return jsonable(
|
|
{
|
|
"id": row.id,
|
|
"criterion_definition_id": row.criterion_definition_id,
|
|
"criterion_key": row.criterion_key,
|
|
"source_id": row.source_id,
|
|
"source_category_id": row.source_category_id,
|
|
"external_filter_key": row.external_filter_key,
|
|
"source_value": row.source_value,
|
|
"encoding": row.encoding,
|
|
"status": row.status,
|
|
"definition_hash": row.definition_hash,
|
|
"created_at": row.created_at,
|
|
"updated_at": row.updated_at,
|
|
}
|
|
)
|
|
|
|
|
|
class DiscoverySearchService:
|
|
def __init__(self, session: Session, adapter: DiscoverySourceAdapter | None = None) -> None:
|
|
self.session = session
|
|
self.adapter = adapter
|
|
self.projects = ProjectService(session)
|
|
self.mappings = CriterionMappingService(session)
|
|
|
|
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
project = self.session.get(BuyingProjectRecord, payload["buying_project_id"])
|
|
if project is None:
|
|
raise AppError(404, "NOT_FOUND", "buying project not found")
|
|
category = self.session.get(SourceCategoryRecord, payload["source_category_id"])
|
|
if category is None:
|
|
raise AppError(404, "NOT_FOUND", "source category not found")
|
|
criteria = self.projects.latest_criteria(project.id)
|
|
if criteria is None:
|
|
raise AppError(422, "CRITERIA_REQUIRED", "buying project has no criteria")
|
|
now = now_utc()
|
|
interval = int(payload.get("schedule_interval_minutes") or 180)
|
|
if interval < 1:
|
|
raise AppError(422, "VALIDATION_ERROR", "schedule_interval_minutes must be > 0")
|
|
existing_active = self.session.scalars(
|
|
select(DiscoverySearchRecord).where(
|
|
DiscoverySearchRecord.buying_project_id == project.id,
|
|
DiscoverySearchRecord.source_id == category.source_id,
|
|
DiscoverySearchRecord.source_category_id == category.id,
|
|
DiscoverySearchRecord.status.in_(
|
|
(SearchStatus.ACTIVE.value, SearchStatus.OUT_OF_SYNC.value, SearchStatus.TOO_BROAD.value)
|
|
),
|
|
)
|
|
).first()
|
|
if existing_active is not None:
|
|
raise AppError(409, "SEARCH_EXISTS", "an active discovery search already exists for this source")
|
|
verified = self.mappings.verified_for(category.id)
|
|
url, _internal = generate_search_url(
|
|
category_url=category.source_url,
|
|
criteria_items=self.projects.criteria_items(project.id, criteria.version),
|
|
mappings=[self._mapping_dict(row) for row in verified],
|
|
)
|
|
search = DiscoverySearchRecord(
|
|
id=uuid4(),
|
|
buying_project_id=project.id,
|
|
source_id=category.source_id,
|
|
source_category_id=category.id,
|
|
criteria_version=criteria.version,
|
|
mapping_version=mapping_version_hash(verified),
|
|
generated_url=url,
|
|
status=SearchStatus.ACTIVE.value,
|
|
schedule_interval_minutes=interval,
|
|
broad_result_threshold=int(payload.get("broad_result_threshold") or 300),
|
|
next_run_at=now,
|
|
last_successful_run_at=None,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
self.session.add(search)
|
|
self.session.flush()
|
|
return self.get(search.id)
|
|
|
|
def generate_query(self, search_id: UUID) -> dict[str, Any]:
|
|
search = self._require_search(search_id)
|
|
project = self.session.get(BuyingProjectRecord, search.buying_project_id)
|
|
criteria = self.projects.latest_criteria(search.buying_project_id)
|
|
category = self.session.get(SourceCategoryRecord, search.source_category_id)
|
|
if project is None or criteria is None or category is None:
|
|
raise AppError(404, "NOT_FOUND", "search dependencies missing")
|
|
verified = self.mappings.verified_for(category.id)
|
|
url, internal_only = generate_search_url(
|
|
category_url=category.source_url,
|
|
criteria_items=self.projects.criteria_items(search.buying_project_id, criteria.version),
|
|
mappings=[self._mapping_dict(row) for row in verified],
|
|
)
|
|
search.criteria_version = criteria.version
|
|
search.mapping_version = mapping_version_hash(verified)
|
|
search.generated_url = url
|
|
if search.status == SearchStatus.OUT_OF_SYNC.value:
|
|
search.status = SearchStatus.ACTIVE.value
|
|
search.next_run_at = now_utc()
|
|
search.updated_at = now_utc()
|
|
self.session.flush()
|
|
body = self.get(search.id)
|
|
body["internal_only"] = internal_only
|
|
return body
|
|
|
|
def get(self, search_id: UUID) -> dict[str, Any]:
|
|
search = self._require_search(search_id)
|
|
return jsonable(
|
|
{
|
|
"id": search.id,
|
|
"buying_project_id": search.buying_project_id,
|
|
"source_id": search.source_id,
|
|
"source_category_id": search.source_category_id,
|
|
"criteria_version": search.criteria_version,
|
|
"mapping_version": search.mapping_version,
|
|
"generated_url": search.generated_url,
|
|
"status": search.status,
|
|
"schedule_interval_minutes": search.schedule_interval_minutes,
|
|
"broad_result_threshold": search.broad_result_threshold,
|
|
"next_run_at": search.next_run_at,
|
|
"last_successful_run_at": search.last_successful_run_at,
|
|
"created_at": search.created_at,
|
|
"updated_at": search.updated_at,
|
|
}
|
|
)
|
|
|
|
def list_for_project(self, project_id: UUID) -> list[dict[str, Any]]:
|
|
if self.session.get(BuyingProjectRecord, project_id) is None:
|
|
raise AppError(404, "NOT_FOUND", "buying project not found")
|
|
rows = self.session.scalars(
|
|
select(DiscoverySearchRecord)
|
|
.where(DiscoverySearchRecord.buying_project_id == project_id)
|
|
.order_by(DiscoverySearchRecord.created_at)
|
|
).all()
|
|
return [self.get(row.id) for row in rows]
|
|
|
|
def patch(self, search_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
|
search = self._require_search(search_id)
|
|
if "schedule_interval_minutes" in payload:
|
|
interval = int(payload["schedule_interval_minutes"])
|
|
if interval < 1:
|
|
raise AppError(422, "VALIDATION_ERROR", "schedule_interval_minutes must be > 0")
|
|
search.schedule_interval_minutes = interval
|
|
if "status" in payload:
|
|
status = str(payload["status"])
|
|
if status not in {SearchStatus.ACTIVE.value, SearchStatus.DISABLED.value}:
|
|
raise AppError(422, "VALIDATION_ERROR", "status may only be ACTIVE or DISABLED")
|
|
search.status = status
|
|
search.updated_at = now_utc()
|
|
self.session.flush()
|
|
return self.get(search.id)
|
|
|
|
def list_all(self) -> list[dict[str, Any]]:
|
|
rows = self.session.scalars(select(DiscoverySearchRecord).order_by(DiscoverySearchRecord.created_at)).all()
|
|
return [self.get(row.id) for row in rows]
|
|
|
|
def list_due(self) -> list[dict[str, Any]]:
|
|
now = now_utc()
|
|
rows = self.session.scalars(
|
|
select(DiscoverySearchRecord)
|
|
.join(BuyingProjectRecord, BuyingProjectRecord.id == DiscoverySearchRecord.buying_project_id)
|
|
.where(
|
|
DiscoverySearchRecord.next_run_at <= now,
|
|
DiscoverySearchRecord.status.in_((SearchStatus.ACTIVE.value, SearchStatus.TOO_BROAD.value)),
|
|
BuyingProjectRecord.status == ProjectStatus.ACTIVE.value,
|
|
)
|
|
.order_by(DiscoverySearchRecord.next_run_at)
|
|
).all()
|
|
return [self.get(row.id) for row in rows]
|
|
|
|
def list_runs(self, search_id: UUID) -> list[dict[str, Any]]:
|
|
self._require_search(search_id)
|
|
rows = self.session.scalars(
|
|
select(SearchRunRecord)
|
|
.where(SearchRunRecord.discovery_search_id == search_id)
|
|
.order_by(SearchRunRecord.started_at.desc())
|
|
).all()
|
|
return [self._run_out(row) for row in rows]
|
|
|
|
def get_run(self, run_id: UUID) -> dict[str, Any]:
|
|
run = self.session.get(SearchRunRecord, run_id)
|
|
if run is None:
|
|
raise AppError(404, "NOT_FOUND", "search run not found")
|
|
items = self.session.scalars(select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == run.id)).all()
|
|
body = self._run_out(run)
|
|
body["items"] = [
|
|
jsonable(
|
|
{
|
|
"id": item.id,
|
|
"source_product_reference_id": item.source_product_reference_id,
|
|
"observation_id": item.observation_id,
|
|
"diff_status": item.diff_status,
|
|
}
|
|
)
|
|
for item in items
|
|
]
|
|
return body
|
|
|
|
def run(self, search_id: UUID) -> dict[str, Any]:
|
|
return SearchExecutionService(self.session, self.adapter).execute(search_id)
|
|
|
|
def _require_search(self, search_id: UUID) -> DiscoverySearchRecord:
|
|
search = self.session.get(DiscoverySearchRecord, search_id)
|
|
if search is None:
|
|
raise AppError(404, "NOT_FOUND", "discovery search not found")
|
|
return search
|
|
|
|
def _mapping_dict(self, row: CriterionMappingRecord) -> dict[str, Any]:
|
|
return {
|
|
"criterion_key": row.criterion_key,
|
|
"external_filter_key": row.external_filter_key,
|
|
"source_value": row.source_value,
|
|
"encoding": row.encoding,
|
|
"status": row.status,
|
|
}
|
|
|
|
def _run_out(self, run: SearchRunRecord) -> dict[str, Any]:
|
|
return jsonable(
|
|
{
|
|
"id": run.id,
|
|
"discovery_search_id": run.discovery_search_id,
|
|
"criteria_version": run.criteria_version,
|
|
"mapping_version": run.mapping_version,
|
|
"generated_url": run.generated_url,
|
|
"started_at": run.started_at,
|
|
"finished_at": run.finished_at,
|
|
"status": run.status,
|
|
"result_count": run.result_count,
|
|
"page_count": run.page_count,
|
|
"new_count": run.new_count,
|
|
"changed_count": run.changed_count,
|
|
"unchanged_count": run.unchanged_count,
|
|
"missing_count": run.missing_count,
|
|
"http_status_summary": run.http_status_summary,
|
|
"error_code": run.error_code,
|
|
}
|
|
)
|
|
|
|
|
|
class SearchExecutionService:
|
|
def __init__(self, session: Session, adapter: DiscoverySourceAdapter | None = None) -> None:
|
|
self.session = session
|
|
self.adapter = adapter
|
|
self.searches = DiscoverySearchService(session, adapter)
|
|
self.projects = ProjectService(session)
|
|
self.mappings = CriterionMappingService(session)
|
|
self.catalog = CatalogRepository(session)
|
|
|
|
def execute(self, search_id: UUID) -> dict[str, Any]:
|
|
search = self.session.get(DiscoverySearchRecord, search_id)
|
|
if search is None:
|
|
raise AppError(404, "NOT_FOUND", "discovery search not found")
|
|
project = self.session.get(BuyingProjectRecord, search.buying_project_id)
|
|
if project is None or project.status != ProjectStatus.ACTIVE.value:
|
|
raise AppError(409, "PROJECT_INACTIVE", "buying project is not active")
|
|
if search.status == SearchStatus.OUT_OF_SYNC.value:
|
|
raise AppError(409, "SEARCH_OUT_OF_SYNC", "generate-query is required before running")
|
|
if search.status in {SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}:
|
|
raise AppError(409, "SEARCH_DISABLED", "discovery search is disabled")
|
|
criteria = self.projects.criteria_version(project.id, search.criteria_version)
|
|
category = self.session.get(SourceCategoryRecord, search.source_category_id)
|
|
if criteria is None or category is None:
|
|
raise AppError(422, "SEARCH_INVALID", "search is missing criteria or category")
|
|
now = now_utc()
|
|
run = SearchRunRecord(
|
|
id=uuid4(),
|
|
discovery_search_id=search.id,
|
|
criteria_version=search.criteria_version,
|
|
mapping_version=search.mapping_version,
|
|
generated_url=search.generated_url,
|
|
started_at=now,
|
|
finished_at=None,
|
|
status=RunStatus.RUNNING.value,
|
|
result_count=0,
|
|
page_count=0,
|
|
new_count=0,
|
|
changed_count=0,
|
|
unchanged_count=0,
|
|
missing_count=0,
|
|
http_status_summary={"pages": []},
|
|
error_code=None,
|
|
)
|
|
self.session.add(run)
|
|
self.session.flush()
|
|
adapter = self.adapter or discovery_adapter(search.source_id)
|
|
pages: list[dict[str, Any]] = []
|
|
items: list[dict[str, Any]] = []
|
|
try:
|
|
pages, items, error_code, advertised = self._paginate(adapter, search)
|
|
except AppError as exc:
|
|
run.status = RunStatus.FAILED.value
|
|
run.error_code = exc.code
|
|
run.finished_at = now_utc()
|
|
search.updated_at = run.finished_at
|
|
self.session.flush()
|
|
return self.searches.get_run(run.id)
|
|
run.page_count = len(pages)
|
|
run.result_count = max(len(items), advertised)
|
|
run.http_status_summary = {"pages": pages}
|
|
run.error_code = error_code
|
|
run.finished_at = now_utc()
|
|
if error_code == "BLOCKED":
|
|
run.status = RunStatus.BLOCKED.value
|
|
search.status = SearchStatus.BLOCKED.value
|
|
search.next_run_at = None
|
|
elif error_code == "BACKOFF":
|
|
run.status = RunStatus.PARTIAL.value
|
|
search.next_run_at = run.finished_at + self._backoff(search.id)
|
|
elif error_code == "PARTIAL":
|
|
run.status = RunStatus.PARTIAL.value
|
|
search.next_run_at = run.finished_at + timedelta(minutes=search.schedule_interval_minutes)
|
|
else:
|
|
run.status = RunStatus.SUCCESS.value
|
|
search.last_successful_run_at = run.finished_at
|
|
search.next_run_at = run.finished_at + timedelta(minutes=search.schedule_interval_minutes)
|
|
if run.result_count > search.broad_result_threshold:
|
|
run.status = RunStatus.TOO_BROAD.value
|
|
search.status = SearchStatus.TOO_BROAD.value
|
|
elif search.status == SearchStatus.TOO_BROAD.value:
|
|
search.status = SearchStatus.ACTIVE.value
|
|
search.updated_at = run.finished_at
|
|
self.session.flush()
|
|
if run.status in {RunStatus.SUCCESS.value, RunStatus.PARTIAL.value, RunStatus.TOO_BROAD.value}:
|
|
self._persist_results(search, run, category, items)
|
|
self._apply_run_counts(run)
|
|
CandidateRankingService(self.session).refresh(project.id)
|
|
self.session.flush()
|
|
return self.searches.get_run(run.id)
|
|
|
|
def _paginate(
|
|
self,
|
|
adapter: DiscoverySourceAdapter,
|
|
search: DiscoverySearchRecord,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None, int]:
|
|
query = {"url": search.generated_url}
|
|
collected: list[dict[str, Any]] = []
|
|
pages: list[dict[str, Any]] = []
|
|
page = 1
|
|
expected_pages = 1
|
|
advertised = 0
|
|
while page <= expected_pages and page <= MAX_PAGES:
|
|
try:
|
|
response = adapter.fetch_results(query, {"page": page})
|
|
except AppError as exc:
|
|
if exc.code == "FIXTURE_NOT_FOUND" and page > 1:
|
|
return pages, collected, "PARTIAL", advertised
|
|
raise
|
|
status = int(response.get("http_status") or 200)
|
|
pages.append({"page": page, "status": status, "result_count": response.get("result_count")})
|
|
if status == 429:
|
|
return pages, collected, "BACKOFF", advertised
|
|
if status == 403 or response.get("challenge") == "captcha" or response.get("blocked"):
|
|
return pages, collected, "BLOCKED", advertised
|
|
if status >= 400:
|
|
return pages, collected, "PARTIAL", advertised
|
|
collected.extend(adapter.parse_results(response))
|
|
if response.get("result_count") is not None:
|
|
advertised = max(advertised, int(response["result_count"]))
|
|
if advertised > search.broad_result_threshold:
|
|
return pages, collected, None, advertised
|
|
expected_pages = int(response.get("page_count") or page)
|
|
page += 1
|
|
if page > MAX_PAGES + 1 or (expected_pages > MAX_PAGES and len(pages) >= MAX_PAGES):
|
|
return pages, collected, "PARTIAL", advertised
|
|
return pages, collected, None, advertised
|
|
|
|
def _backoff(self, search_id: UUID) -> timedelta:
|
|
rows = self.session.scalars(
|
|
select(SearchRunRecord)
|
|
.where(SearchRunRecord.discovery_search_id == search_id)
|
|
.order_by(SearchRunRecord.started_at.desc())
|
|
.limit(8)
|
|
).all()
|
|
streak = 0
|
|
for row in rows:
|
|
if row.error_code == "BACKOFF":
|
|
streak += 1
|
|
continue
|
|
break
|
|
return timedelta(minutes=BACKOFF_BASE_MINUTES * (2 ** min(streak, 5)))
|
|
|
|
def _persist_results(
|
|
self,
|
|
search: DiscoverySearchRecord,
|
|
run: SearchRunRecord,
|
|
category: SourceCategoryRecord,
|
|
items: list[dict[str, Any]],
|
|
) -> None:
|
|
adapter = self.adapter or discovery_adapter(search.source_id)
|
|
known_rows = list(
|
|
self.session.scalars(
|
|
select(SourceProductReferenceRecord).where(
|
|
SourceProductReferenceRecord.source_id == search.source_id,
|
|
SourceProductReferenceRecord.source_category_id == category.id,
|
|
)
|
|
)
|
|
)
|
|
known = [(row.id, self._signals_from_ref(row)) for row in known_rows]
|
|
current_ids: set[UUID] = set()
|
|
current_obs: dict[UUID, dict[str, Any]] = {}
|
|
observations: dict[UUID, ProductObservationRecord] = {}
|
|
for raw in items:
|
|
normalized = adapter.normalize_result(raw)
|
|
incoming = self._signals_from_norm(normalized)
|
|
decision = decide_identity(incoming, known)
|
|
if decision.match_status == MatchStatus.MATCHED and decision.existing_id is not None:
|
|
ref = self.session.get(SourceProductReferenceRecord, decision.existing_id)
|
|
assert ref is not None
|
|
self._touch_ref(ref, normalized, incoming, MatchStatus.MATCHED)
|
|
else:
|
|
ref = self._new_ref(search, category, normalized, incoming, decision.match_status)
|
|
known.append((ref.id, incoming))
|
|
known_rows.append(ref)
|
|
current_ids.add(ref.id)
|
|
obs = ProductObservationRecord(
|
|
id=uuid4(),
|
|
source_product_reference_id=ref.id,
|
|
search_run_id=run.id,
|
|
observed_at=run.finished_at or now_utc(),
|
|
raw_title=str(normalized.get("source_title") or ""),
|
|
best_price=None if normalized.get("best_price") is None else Decimal(str(normalized["best_price"])),
|
|
currency=normalized.get("currency"),
|
|
availability=normalized.get("availability"),
|
|
seller_count=normalized.get("seller_count"),
|
|
)
|
|
self.session.add(obs)
|
|
self.session.flush()
|
|
_store_observation_attributes(
|
|
self.session,
|
|
obs.id,
|
|
normalized.get("raw_attributes") or {},
|
|
normalized.get("normalized_attributes") or {},
|
|
)
|
|
observations[ref.id] = obs
|
|
current_obs[ref.id] = _obs_dict(self.session, obs)
|
|
self._upsert_candidate(search.buying_project_id, ref, decision.match_status)
|
|
self.session.flush()
|
|
previous_ids, previous_obs = self._previous_success(search.id, run.id)
|
|
diffs = diff_run(
|
|
previous_ids=previous_ids,
|
|
current_ids=current_ids,
|
|
previous_obs=previous_obs,
|
|
current_obs=current_obs,
|
|
run_status=run.status,
|
|
)
|
|
for ref_id, status in diffs.items():
|
|
obs = observations.get(ref_id)
|
|
self.session.add(
|
|
SearchRunItemRecord(
|
|
id=uuid4(),
|
|
search_run_id=run.id,
|
|
source_product_reference_id=ref_id,
|
|
observation_id=None if obs is None else obs.id,
|
|
diff_status=status.value,
|
|
)
|
|
)
|
|
ref = self.session.get(SourceProductReferenceRecord, ref_id)
|
|
if ref is None:
|
|
continue
|
|
missing = run.status == RunStatus.SUCCESS.value and status == DiffStatus.MISSING
|
|
reappeared = ref_id in current_ids and ref.lifecycle_status != LifecycleStatus.ACTIVE.value
|
|
lifecycle, streak = next_lifecycle(
|
|
current=ref.lifecycle_status,
|
|
missing_in_success_run=missing,
|
|
consecutive_missing=ref.consecutive_missing_success_runs,
|
|
reappeared=reappeared or (ref_id in current_ids and status != DiffStatus.MISSING),
|
|
)
|
|
if ref.lifecycle_status == LifecycleStatus.ARCHIVED.value:
|
|
continue
|
|
if ref_id in current_ids:
|
|
ref.lifecycle_status = LifecycleStatus.ACTIVE.value
|
|
ref.consecutive_missing_success_runs = 0
|
|
else:
|
|
ref.lifecycle_status = lifecycle
|
|
ref.consecutive_missing_success_runs = streak
|
|
|
|
def _apply_run_counts(self, run: SearchRunRecord) -> None:
|
|
items = self.session.scalars(
|
|
select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == run.id)
|
|
).all()
|
|
run.new_count = sum(1 for item in items if item.diff_status == DiffStatus.NEW.value)
|
|
run.changed_count = sum(1 for item in items if item.diff_status == DiffStatus.CHANGED.value)
|
|
run.unchanged_count = sum(1 for item in items if item.diff_status == DiffStatus.UNCHANGED.value)
|
|
run.missing_count = sum(1 for item in items if item.diff_status == DiffStatus.MISSING.value)
|
|
|
|
def _previous_success(self, search_id: UUID, current_run_id: UUID) -> tuple[set[UUID], dict[UUID, dict[str, Any]]]:
|
|
runs = self.session.scalars(
|
|
select(SearchRunRecord)
|
|
.where(
|
|
SearchRunRecord.discovery_search_id == search_id,
|
|
SearchRunRecord.status == RunStatus.SUCCESS.value,
|
|
SearchRunRecord.id != current_run_id,
|
|
)
|
|
.order_by(SearchRunRecord.started_at.asc())
|
|
).all()
|
|
known: set[UUID] = set()
|
|
latest_obs: dict[UUID, dict[str, Any]] = {}
|
|
for run in runs:
|
|
items = self.session.scalars(
|
|
select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == run.id)
|
|
).all()
|
|
known.update(
|
|
item.source_product_reference_id for item in items if item.diff_status != DiffStatus.MISSING.value
|
|
)
|
|
for row in self.session.scalars(
|
|
select(ProductObservationRecord).where(ProductObservationRecord.search_run_id == run.id)
|
|
):
|
|
latest_obs[row.source_product_reference_id] = _obs_dict(self.session, row)
|
|
return known, latest_obs
|
|
|
|
def _new_ref(
|
|
self,
|
|
search: DiscoverySearchRecord,
|
|
category: SourceCategoryRecord,
|
|
normalized: dict[str, Any],
|
|
incoming: IdentitySignals,
|
|
match_status: MatchStatus,
|
|
) -> SourceProductReferenceRecord:
|
|
now = now_utc()
|
|
ref = SourceProductReferenceRecord(
|
|
id=uuid4(),
|
|
source_id=search.source_id,
|
|
source_category_id=category.id,
|
|
source_external_id=normalized.get("source_external_id"),
|
|
source_url=normalized.get("source_url"),
|
|
source_title=str(normalized.get("source_title") or ""),
|
|
manufacturer=normalized.get("manufacturer"),
|
|
model=normalized.get("model"),
|
|
manufacturer_sku=normalized.get("manufacturer_sku"),
|
|
gtin_ean=normalized.get("gtin_ean"),
|
|
normalized_spec_fingerprint=incoming.fingerprint,
|
|
first_seen_at=now,
|
|
last_seen_at=now,
|
|
lifecycle_status=LifecycleStatus.ACTIVE.value,
|
|
match_status=match_status.value,
|
|
consecutive_missing_success_runs=0,
|
|
product_variant_id=None,
|
|
)
|
|
self.session.add(ref)
|
|
self.session.flush()
|
|
return ref
|
|
|
|
def _touch_ref(
|
|
self,
|
|
ref: SourceProductReferenceRecord,
|
|
normalized: dict[str, Any],
|
|
incoming: IdentitySignals,
|
|
match_status: MatchStatus,
|
|
) -> None:
|
|
ref.source_title = str(normalized.get("source_title") or ref.source_title)
|
|
ref.source_url = normalized.get("source_url") or ref.source_url
|
|
ref.manufacturer = normalized.get("manufacturer") or ref.manufacturer
|
|
ref.model = normalized.get("model") or ref.model
|
|
ref.manufacturer_sku = normalized.get("manufacturer_sku") or ref.manufacturer_sku
|
|
ref.gtin_ean = normalized.get("gtin_ean") or ref.gtin_ean
|
|
ref.normalized_spec_fingerprint = incoming.fingerprint
|
|
ref.last_seen_at = now_utc()
|
|
ref.match_status = match_status.value
|
|
if ref.lifecycle_status != LifecycleStatus.ARCHIVED.value:
|
|
ref.lifecycle_status = LifecycleStatus.ACTIVE.value
|
|
ref.consecutive_missing_success_runs = 0
|
|
|
|
def _upsert_candidate(self, project_id: UUID, ref: SourceProductReferenceRecord, match_status: MatchStatus) -> None:
|
|
existing = self.session.scalars(
|
|
select(ProjectCandidateRecord).where(
|
|
ProjectCandidateRecord.buying_project_id == project_id,
|
|
ProjectCandidateRecord.source_product_reference_id == ref.id,
|
|
)
|
|
).first()
|
|
now = now_utc()
|
|
if existing is None:
|
|
candidate = ProjectCandidateRecord(
|
|
id=uuid4(),
|
|
buying_project_id=project_id,
|
|
source_product_reference_id=ref.id,
|
|
product_variant_id=ref.product_variant_id,
|
|
status=CandidateStatus.CANDIDATE.value,
|
|
match_status=match_status.value,
|
|
product_eligibility=None,
|
|
product_score=None,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
self.session.add(candidate)
|
|
self.session.flush()
|
|
self.session.add(
|
|
CandidateStatusHistoryRecord(
|
|
id=uuid4(),
|
|
candidate_id=candidate.id,
|
|
from_status=None,
|
|
to_status=candidate.status,
|
|
reason="discovered",
|
|
created_at=now,
|
|
)
|
|
)
|
|
return
|
|
existing.match_status = match_status.value
|
|
existing.product_variant_id = ref.product_variant_id
|
|
existing.updated_at = now
|
|
|
|
def _signals_from_norm(self, normalized: dict[str, Any]) -> IdentitySignals:
|
|
attrs = dict(normalized.get("normalized_attributes") or {})
|
|
if normalized.get("manufacturer"):
|
|
attrs.setdefault("manufacturer", normalized["manufacturer"])
|
|
if normalized.get("model"):
|
|
attrs.setdefault("model", normalized["model"])
|
|
return IdentitySignals(
|
|
source_external_id=normalized.get("source_external_id"),
|
|
gtin_ean=normalized.get("gtin_ean"),
|
|
manufacturer_sku=normalized.get("manufacturer_sku"),
|
|
source_url=normalized.get("source_url"),
|
|
title=str(normalized.get("source_title") or ""),
|
|
manufacturer=normalized.get("manufacturer"),
|
|
model=normalized.get("model"),
|
|
fingerprint=spec_fingerprint(attrs),
|
|
)
|
|
|
|
def _signals_from_ref(self, row: SourceProductReferenceRecord) -> IdentitySignals:
|
|
return IdentitySignals(
|
|
source_external_id=row.source_external_id,
|
|
gtin_ean=row.gtin_ean,
|
|
manufacturer_sku=row.manufacturer_sku,
|
|
source_url=row.source_url,
|
|
title=row.source_title,
|
|
manufacturer=row.manufacturer,
|
|
model=row.model,
|
|
fingerprint=row.normalized_spec_fingerprint or "",
|
|
)
|
|
|
|
|
|
class CandidateRankingService:
|
|
def __init__(self, session: Session) -> None:
|
|
self.session = session
|
|
self.catalog = CatalogRepository(session)
|
|
|
|
def refresh(self, project_id: UUID) -> list[dict[str, Any]]:
|
|
project = self.session.get(BuyingProjectRecord, project_id)
|
|
if project is None:
|
|
raise AppError(404, "NOT_FOUND", "buying project not found")
|
|
profile = None
|
|
if project.template_id is not None and project.template_version is not None:
|
|
profile = self.catalog.get_profile(project.template_id, project.template_version)
|
|
model = self.catalog.get_scoring_model(profile.scoring_version) if profile is not None else None
|
|
if profile is None or model is None:
|
|
return self.list_candidates(project_id)
|
|
definitions = self.catalog.all_definitions()
|
|
candidates = self.session.scalars(
|
|
select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project_id)
|
|
).all()
|
|
payload: list[dict[str, Any]] = []
|
|
for candidate in candidates:
|
|
ref = self.session.get(SourceProductReferenceRecord, candidate.source_product_reference_id)
|
|
if ref is None or ref.lifecycle_status == LifecycleStatus.ARCHIVED.value:
|
|
continue
|
|
obs = self.session.scalars(
|
|
select(ProductObservationRecord)
|
|
.where(ProductObservationRecord.source_product_reference_id == ref.id)
|
|
.order_by(ProductObservationRecord.observed_at.desc())
|
|
).first()
|
|
payload.append(
|
|
{
|
|
"candidate_id": candidate.id,
|
|
"source_product_reference_id": ref.id,
|
|
"manufacturer": None if ref is None else ref.manufacturer,
|
|
"model": None if ref is None else ref.model,
|
|
"source_title": None if ref is None else ref.source_title,
|
|
"normalized_attributes": {} if obs is None else _attribute_map(self.session, obs.id),
|
|
"lifecycle_status": ref.lifecycle_status,
|
|
"match_status": candidate.match_status,
|
|
"best_price": None if obs is None else obs.best_price,
|
|
}
|
|
)
|
|
ranked = rank_candidates(
|
|
profile=profile,
|
|
scoring_model=model,
|
|
definitions=definitions,
|
|
candidates=payload,
|
|
shortlist_size=project.shortlist_size,
|
|
as_of=now_utc(),
|
|
)
|
|
by_id = {item["candidate_id"]: item for item in ranked}
|
|
now = now_utc()
|
|
for candidate in candidates:
|
|
item = by_id.get(candidate.id)
|
|
if item is None:
|
|
continue
|
|
new_status = item["candidate_status"]
|
|
if candidate.status != new_status:
|
|
self.session.add(
|
|
CandidateStatusHistoryRecord(
|
|
id=uuid4(),
|
|
candidate_id=candidate.id,
|
|
from_status=candidate.status,
|
|
to_status=new_status,
|
|
reason="ranking",
|
|
created_at=now,
|
|
)
|
|
)
|
|
candidate.status = new_status
|
|
candidate.product_eligibility = item["product_eligibility"]
|
|
candidate.product_score = item["product_score"]
|
|
candidate.updated_at = now
|
|
self.session.flush()
|
|
return ranked
|
|
|
|
def list_candidates(self, project_id: UUID, shortlisted_only: bool = False) -> list[dict[str, Any]]:
|
|
project = self.session.get(BuyingProjectRecord, project_id)
|
|
if project is None:
|
|
raise AppError(404, "NOT_FOUND", "buying project not found")
|
|
stmt = select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project_id)
|
|
if shortlisted_only:
|
|
stmt = stmt.where(ProjectCandidateRecord.status == CandidateStatus.SHORTLISTED.value)
|
|
rows = self.session.scalars(stmt.order_by(ProjectCandidateRecord.product_score.desc().nullslast())).all()
|
|
return [self._candidate_summary(project, row) for row in rows]
|
|
|
|
def get_candidate(self, project_id: UUID, candidate_id: UUID) -> dict[str, Any]:
|
|
project = self.session.get(BuyingProjectRecord, project_id)
|
|
candidate = self.session.get(ProjectCandidateRecord, candidate_id)
|
|
if project is None or candidate is None or candidate.buying_project_id != project_id:
|
|
raise AppError(404, "NOT_FOUND", "candidate not found")
|
|
summary = self._candidate_summary(project, candidate)
|
|
ref = self.session.get(SourceProductReferenceRecord, candidate.source_product_reference_id)
|
|
obs = None
|
|
if ref is not None:
|
|
obs = self.session.scalars(
|
|
select(ProductObservationRecord)
|
|
.where(ProductObservationRecord.source_product_reference_id == ref.id)
|
|
.order_by(ProductObservationRecord.observed_at.desc())
|
|
).first()
|
|
summary.update(
|
|
{
|
|
"normalized_attributes": {} if obs is None else _attribute_map(self.session, obs.id),
|
|
"source": None
|
|
if ref is None
|
|
else {
|
|
"source_id": ref.source_id,
|
|
"source_category_id": ref.source_category_id,
|
|
"source_external_id": ref.source_external_id,
|
|
"source_url": ref.source_url,
|
|
},
|
|
"current_observation": None if obs is None else _observation_out(self.session, obs),
|
|
}
|
|
)
|
|
return jsonable(summary)
|
|
|
|
def list_observations(self, project_id: UUID, candidate_id: UUID) -> list[dict[str, Any]]:
|
|
candidate = self.session.get(ProjectCandidateRecord, candidate_id)
|
|
if candidate is None or candidate.buying_project_id != project_id:
|
|
raise AppError(404, "NOT_FOUND", "candidate not found")
|
|
rows = self.session.scalars(
|
|
select(ProductObservationRecord)
|
|
.where(ProductObservationRecord.source_product_reference_id == candidate.source_product_reference_id)
|
|
.order_by(ProductObservationRecord.observed_at.desc())
|
|
).all()
|
|
return [_observation_out(self.session, row) for row in rows]
|
|
|
|
def _candidate_summary(self, project: BuyingProjectRecord, row: ProjectCandidateRecord) -> dict[str, Any]:
|
|
ref = self.session.get(SourceProductReferenceRecord, row.source_product_reference_id)
|
|
obs = None
|
|
last_item = None
|
|
if ref is not None:
|
|
obs = self.session.scalars(
|
|
select(ProductObservationRecord)
|
|
.where(ProductObservationRecord.source_product_reference_id == ref.id)
|
|
.order_by(ProductObservationRecord.observed_at.desc())
|
|
).first()
|
|
last_item = self.session.scalars(
|
|
select(SearchRunItemRecord)
|
|
.where(SearchRunItemRecord.source_product_reference_id == ref.id)
|
|
.order_by(SearchRunItemRecord.id.desc())
|
|
).first()
|
|
ranking_status = (
|
|
RankingStatus.NOT_CONFIGURED.value if project.template_id is None else RankingStatus.CONFIGURED.value
|
|
)
|
|
return jsonable(
|
|
{
|
|
"id": row.id,
|
|
"candidate_id": row.id,
|
|
"buying_project_id": row.buying_project_id,
|
|
"source_product_reference_id": row.source_product_reference_id,
|
|
"product_variant_id": row.product_variant_id,
|
|
"title": None if ref is None else ref.source_title,
|
|
"source_title": None if ref is None else ref.source_title,
|
|
"manufacturer": None if ref is None else ref.manufacturer,
|
|
"model": None if ref is None else ref.model,
|
|
"source_url": None if ref is None else ref.source_url,
|
|
"current_best_price": None if obs is None else obs.best_price,
|
|
"currency": None if obs is None else obs.currency,
|
|
"current_dynamic_attributes": [] if obs is None else _dynamic_attributes(self.session, obs.id),
|
|
"product_score": row.product_score,
|
|
"product_eligibility": row.product_eligibility,
|
|
"ranking_status": ranking_status,
|
|
"shortlist_status": row.status,
|
|
"status": row.status,
|
|
"current_diff_status": None if last_item is None else last_item.diff_status,
|
|
"product_lifecycle_status": None if ref is None else ref.lifecycle_status,
|
|
"lifecycle_status": None if ref is None else ref.lifecycle_status,
|
|
"match_status": row.match_status,
|
|
"first_seen_at": None if ref is None else ref.first_seen_at,
|
|
"last_seen_at": None if ref is None else ref.last_seen_at,
|
|
"created_at": row.created_at,
|
|
"updated_at": row.updated_at,
|
|
}
|
|
)
|
|
|
|
|
|
def _attribute_map(session: Session, observation_id: UUID) -> dict[str, Any]:
|
|
rows = session.scalars(
|
|
select(ProductObservationAttributeRecord).where(
|
|
ProductObservationAttributeRecord.observation_id == observation_id
|
|
)
|
|
).all()
|
|
mapped: dict[str, Any] = {}
|
|
for row in rows:
|
|
definition = None if row.criterion_definition_id is None else session.get(CriterionDefinitionRecord, row.criterion_definition_id)
|
|
key = row.attribute_key
|
|
if definition is not None and definition.scoring_attribute_key:
|
|
key = definition.scoring_attribute_key
|
|
mapped[key] = row.normalized_value
|
|
return mapped
|
|
|
|
|
|
def _dynamic_attributes(session: Session, observation_id: UUID) -> list[dict[str, Any]]:
|
|
rows = session.scalars(
|
|
select(ProductObservationAttributeRecord).where(
|
|
ProductObservationAttributeRecord.observation_id == observation_id
|
|
)
|
|
).all()
|
|
return [
|
|
{
|
|
"criterion_definition_id": row.criterion_definition_id,
|
|
"raw_name": row.raw_name or row.attribute_key,
|
|
"raw_value": row.raw_value,
|
|
"normalized_value": row.normalized_value,
|
|
"unit": row.unit,
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def _store_observation_attributes(
|
|
session: Session,
|
|
observation_id: UUID,
|
|
raw_attributes: dict[str, Any],
|
|
normalized_attributes: dict[str, Any],
|
|
) -> None:
|
|
keys = set(raw_attributes) | set(normalized_attributes)
|
|
definitions = {
|
|
row.scoring_attribute_key or row.key: row
|
|
for row in session.scalars(select(CriterionDefinitionRecord)).all()
|
|
if row.scoring_attribute_key or row.key
|
|
}
|
|
attr_maps = {
|
|
row.raw_name: row
|
|
for row in session.scalars(select(SourceAttributeMappingRecord)).all()
|
|
}
|
|
for key in keys:
|
|
mapping = attr_maps.get(key)
|
|
definition = None if mapping is None else session.get(CriterionDefinitionRecord, mapping.criterion_definition_id)
|
|
if definition is None:
|
|
definition = definitions.get(key)
|
|
session.add(
|
|
ProductObservationAttributeRecord(
|
|
id=uuid4(),
|
|
observation_id=observation_id,
|
|
criterion_definition_id=None if definition is None else definition.id,
|
|
attribute_key=key,
|
|
raw_name=key,
|
|
raw_value=raw_attributes.get(key),
|
|
normalized_value=normalized_attributes.get(key),
|
|
unit=None if definition is None else definition.unit,
|
|
)
|
|
)
|
|
|
|
|
|
def _obs_dict(session: Session, row: ProductObservationRecord) -> dict[str, Any]:
|
|
return {
|
|
"best_price": row.best_price,
|
|
"availability": row.availability,
|
|
"seller_count": row.seller_count,
|
|
"normalized_attributes": _attribute_map(session, row.id),
|
|
}
|
|
|
|
|
|
def _observation_out(session: Session, row: ProductObservationRecord) -> dict[str, Any]:
|
|
return jsonable(
|
|
{
|
|
"observation_id": row.id,
|
|
"observed_at": row.observed_at,
|
|
"best_price": row.best_price,
|
|
"currency": row.currency,
|
|
"availability": row.availability,
|
|
"seller_count": row.seller_count,
|
|
"dynamic_attributes": _dynamic_attributes(session, row.id),
|
|
"search_run_id": row.search_run_id,
|
|
}
|
|
)
|
|
|
|
|
|
class SourceAttributeMappingService:
|
|
def __init__(self, session: Session) -> None:
|
|
self.session = session
|
|
|
|
def list(self) -> list[dict[str, Any]]:
|
|
rows = self.session.scalars(select(SourceAttributeMappingRecord).order_by(SourceAttributeMappingRecord.raw_name)).all()
|
|
return [self._out(row) for row in rows]
|
|
|
|
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
definition = get_definition(self.session, payload["criterion_definition_id"])
|
|
if definition is None:
|
|
raise AppError(404, "NOT_FOUND", "criterion definition not found")
|
|
now = now_utc()
|
|
row = SourceAttributeMappingRecord(
|
|
id=uuid4(),
|
|
source_id=str(payload["source_id"]),
|
|
source_category_id=payload.get("source_category_id"),
|
|
raw_name=str(payload["raw_name"]),
|
|
criterion_definition_id=definition.id,
|
|
status=str(payload.get("status") or MappingStatus.LEARNED_UNVERIFIED.value),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
self.session.add(row)
|
|
self.session.flush()
|
|
return self._out(row)
|
|
|
|
def patch(self, mapping_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
|
row = self.session.get(SourceAttributeMappingRecord, mapping_id)
|
|
if row is None:
|
|
raise AppError(404, "NOT_FOUND", "source attribute mapping not found")
|
|
if "raw_name" in payload:
|
|
row.raw_name = str(payload["raw_name"])
|
|
if "criterion_definition_id" in payload:
|
|
row.criterion_definition_id = payload["criterion_definition_id"]
|
|
if "status" in payload:
|
|
row.status = str(payload["status"])
|
|
row.updated_at = now_utc()
|
|
self.session.flush()
|
|
return self._out(row)
|
|
|
|
def _out(self, row: SourceAttributeMappingRecord) -> dict[str, Any]:
|
|
return jsonable(
|
|
{
|
|
"id": row.id,
|
|
"source_id": row.source_id,
|
|
"source_category_id": row.source_category_id,
|
|
"raw_name": row.raw_name,
|
|
"criterion_definition_id": row.criterion_definition_id,
|
|
"status": row.status,
|
|
"created_at": row.created_at,
|
|
"updated_at": row.updated_at,
|
|
}
|
|
)
|
|
|