Stelle den Frontend-MVP und die v0.3-API auf develop bereit, damit Dev API und PWA zusammen deployen.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
827e6a2a11
commit
b276758ee1
|
|
@ -10,6 +10,9 @@ ENVIRONMENT=production
|
||||||
ENGINE_BUILD=dev
|
ENGINE_BUILD=dev
|
||||||
BUYING_AGENT_API_BIND=127.0.0.1
|
BUYING_AGENT_API_BIND=127.0.0.1
|
||||||
BUYING_AGENT_API_PORT=8000
|
BUYING_AGENT_API_PORT=8000
|
||||||
|
BUYING_AGENT_FRONTEND_BIND=127.0.0.1
|
||||||
|
BUYING_AGENT_FRONTEND_PORT=4173
|
||||||
|
CORS_ALLOWED_ORIGINS=http://localhost:4173,http://127.0.0.1:4173,http://localhost:5173,http://127.0.0.1:5173
|
||||||
|
|
||||||
# Production on the Raspberry Pi (192.168.2.49):
|
# Production on the Raspberry Pi (192.168.2.49):
|
||||||
# BUYING_AGENT_API_BIND=0.0.0.0
|
# BUYING_AGENT_API_BIND=0.0.0.0
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,13 @@ jobs:
|
||||||
export ENGINE_BUILD BUYING_AGENT_API_PORT
|
export ENGINE_BUILD BUYING_AGENT_API_PORT
|
||||||
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
||||||
BUYING_AGENT_API_PORT=8095
|
BUYING_AGENT_API_PORT=8095
|
||||||
docker compose -f docker-compose.dev-env.yml build --pull=false api
|
if ! grep -q '^CORS_ALLOWED_ORIGINS=' .env; then
|
||||||
|
echo 'CORS_ALLOWED_ORIGINS=http://192.168.2.49:5174,http://localhost:5173' >> .env
|
||||||
|
fi
|
||||||
|
if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then
|
||||||
|
echo 'BUYING_AGENT_FRONTEND_PORT=5174' >> .env
|
||||||
|
fi
|
||||||
|
docker compose -f docker-compose.dev-env.yml build --pull=false api frontend
|
||||||
if ! ENGINE_BUILD="$ENGINE_BUILD" BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
if ! ENGINE_BUILD="$ENGINE_BUILD" BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
||||||
docker compose -f docker-compose.dev-env.yml up -d --wait; then
|
docker compose -f docker-compose.dev-env.yml up -d --wait; then
|
||||||
echo "compose up --wait failed — api logs:"
|
echo "compose up --wait failed — api logs:"
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,13 @@ jobs:
|
||||||
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
||||||
BUYING_AGENT_API_BIND=0.0.0.0
|
BUYING_AGENT_API_BIND=0.0.0.0
|
||||||
BUYING_AGENT_API_PORT=8006
|
BUYING_AGENT_API_PORT=8006
|
||||||
docker compose build --pull=false api
|
if ! grep -q '^CORS_ALLOWED_ORIGINS=' .env; then
|
||||||
|
echo 'CORS_ALLOWED_ORIGINS=http://192.168.2.49:4173,http://localhost:4173' >> .env
|
||||||
|
fi
|
||||||
|
if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then
|
||||||
|
echo 'BUYING_AGENT_FRONTEND_PORT=4173' >> .env
|
||||||
|
fi
|
||||||
|
docker compose build --pull=false api frontend
|
||||||
if ! ENGINE_BUILD="$ENGINE_BUILD" \
|
if ! ENGINE_BUILD="$ENGINE_BUILD" \
|
||||||
BUYING_AGENT_API_BIND="$BUYING_AGENT_API_BIND" \
|
BUYING_AGENT_API_BIND="$BUYING_AGENT_API_BIND" \
|
||||||
BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
||||||
|
|
|
||||||
|
|
@ -57,3 +57,18 @@ jobs:
|
||||||
curl -sf http://localhost:8095/health/ready
|
curl -sf http://localhost:8095/health/ready
|
||||||
echo
|
echo
|
||||||
echo "LAN smoke OK: http://192.168.2.49:8095"
|
echo "LAN smoke OK: http://192.168.2.49:8095"
|
||||||
|
|
||||||
|
frontend-checks:
|
||||||
|
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Frontend typecheck, lint, test, build
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
APP_DIR="$HOME/docker/buyingagent-dev"
|
||||||
|
cd "$APP_DIR/frontend"
|
||||||
|
npm ci || npm install
|
||||||
|
npm run typecheck
|
||||||
|
npm run lint
|
||||||
|
npm test
|
||||||
|
npm run build
|
||||||
|
|
|
||||||
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -14,6 +14,10 @@
|
||||||
dist/
|
dist/
|
||||||
build/
|
build/
|
||||||
|
|
||||||
|
# Frontend
|
||||||
|
node_modules/
|
||||||
|
frontend/dev-dist/
|
||||||
|
|
||||||
# OS / editor
|
# OS / editor
|
||||||
.DS_Store
|
.DS_Store
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
|
|
|
||||||
377
.specs/buying-agent-architecture-addendum-v1.md
Normal file
377
.specs/buying-agent-architecture-addendum-v1.md
Normal file
|
|
@ -0,0 +1,377 @@
|
||||||
|
# Buying Agent – Architecture Addendum v1
|
||||||
|
|
||||||
|
**Status:** ergänzende Architekturvorgabe
|
||||||
|
**Gültig zusammen mit:** `buying-agent-v0.1-spec-v1.1.md`
|
||||||
|
**Stand:** 2026-10-03
|
||||||
|
|
||||||
|
## 1. Zweck
|
||||||
|
|
||||||
|
Dieses Addendum ergänzt die bestehende Spezifikation um übergreifende Architekturentscheidungen für das spätere Zielsystem.
|
||||||
|
|
||||||
|
Es ersetzt **nicht** die fachliche Spezifikation `buying-agent-v0.1-spec-v1.1.md`.
|
||||||
|
|
||||||
|
Bei Konflikten zwischen beiden Dokumenten hat dieses Addendum für die hier behandelten Architekturthemen Vorrang.
|
||||||
|
|
||||||
|
Ziel ist, den aktuellen v0.1-Slice schlank zu halten, ohne spätere Erweiterungen auf Frontend, mehrere Kaufprojekte und mehrere Benutzer unnötig zu erschweren.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Zielstruktur des Repositories
|
||||||
|
|
||||||
|
Das Projekt wird als Monorepo strukturiert:
|
||||||
|
|
||||||
|
```text
|
||||||
|
buying-agent/
|
||||||
|
├── README.md
|
||||||
|
├── docker-compose.yml
|
||||||
|
├── .env.example
|
||||||
|
├── .gitignore
|
||||||
|
├── docs/
|
||||||
|
│ ├── buying-agent-v0.1-spec-v1.1.md
|
||||||
|
│ └── buying-agent-architecture-addendum-v1.md
|
||||||
|
├── backend/
|
||||||
|
│ ├── Dockerfile
|
||||||
|
│ ├── pyproject.toml
|
||||||
|
│ ├── uv.lock
|
||||||
|
│ ├── alembic.ini
|
||||||
|
│ ├── migrations/
|
||||||
|
│ ├── app/
|
||||||
|
│ ├── rules/
|
||||||
|
│ ├── profiles/
|
||||||
|
│ ├── fixtures/
|
||||||
|
│ ├── scripts/
|
||||||
|
│ └── tests/
|
||||||
|
└── frontend/
|
||||||
|
└── README.md
|
||||||
|
```
|
||||||
|
|
||||||
|
### Regeln
|
||||||
|
|
||||||
|
- `docker-compose.yml` bleibt im Repository-Root.
|
||||||
|
- Das Backend ist der aktuell implementierte v0.1-Scope.
|
||||||
|
- `frontend/` wird in v0.1 **nicht implementiert**.
|
||||||
|
- `frontend/README.md` dokumentiert lediglich, dass dort später die Benutzeroberfläche entsteht.
|
||||||
|
- Keine zusätzliche `infra/`-Struktur in v0.1, solange sie nicht konkret benötigt wird.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Backend-/Frontend-Grenze
|
||||||
|
|
||||||
|
Das Backend bleibt das **System of Record** und besitzt:
|
||||||
|
|
||||||
|
- Domain-Modell
|
||||||
|
- Constraints
|
||||||
|
- Scoring
|
||||||
|
- Evaluation
|
||||||
|
- Persistenz
|
||||||
|
- Preis- und Produktlogik
|
||||||
|
- zukünftige Kaufentscheidungslogik
|
||||||
|
|
||||||
|
Das spätere Frontend konsumiert ausschließlich die versionierte Backend-API, primär unter:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/api/v1
|
||||||
|
```
|
||||||
|
|
||||||
|
Das Frontend enthält keine fachlich verbindliche Business-Logik.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Zukünftige Multi-User-Architektur
|
||||||
|
|
||||||
|
Das Zielsystem soll später von Familie und gegebenenfalls engen Freunden genutzt werden können.
|
||||||
|
|
||||||
|
Die Architektur darf daher nicht von einem impliziten Single-User-Modell ausgehen.
|
||||||
|
|
||||||
|
Zielmodell:
|
||||||
|
|
||||||
|
```text
|
||||||
|
User
|
||||||
|
↕
|
||||||
|
WorkspaceMember
|
||||||
|
↕
|
||||||
|
Workspace
|
||||||
|
↓
|
||||||
|
BuyingProject
|
||||||
|
↓
|
||||||
|
ProjectCandidate
|
||||||
|
↓
|
||||||
|
ProductVariant
|
||||||
|
```
|
||||||
|
|
||||||
|
Ein Benutzer kann Mitglied in mehreren Workspaces sein.
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Lars
|
||||||
|
├── Privat
|
||||||
|
├── Familie
|
||||||
|
└── Freunde
|
||||||
|
```
|
||||||
|
|
||||||
|
### Noch nicht Bestandteil von v0.1
|
||||||
|
|
||||||
|
Nicht implementieren:
|
||||||
|
|
||||||
|
- Login
|
||||||
|
- Authentifizierung
|
||||||
|
- Passwortverwaltung
|
||||||
|
- Einladungen
|
||||||
|
- RBAC
|
||||||
|
- komplexe ACLs
|
||||||
|
- Sharing-UI
|
||||||
|
- Workspace-Administration
|
||||||
|
|
||||||
|
Die aktuelle Architektur darf deren spätere Einführung aber nicht unnötig erschweren.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Globale versus Workspace-bezogene Daten
|
||||||
|
|
||||||
|
### Global wiederverwendbare Daten
|
||||||
|
|
||||||
|
Diese Entitäten bleiben systemweit gemeinsam nutzbar:
|
||||||
|
|
||||||
|
- Product
|
||||||
|
- ProductVariant
|
||||||
|
- Offer
|
||||||
|
- PriceObservation
|
||||||
|
- AttributeDefinition
|
||||||
|
- Source
|
||||||
|
|
||||||
|
Sie erhalten **keine** `user_id` oder `workspace_id`.
|
||||||
|
|
||||||
|
Ein Produkt oder Angebot darf nicht pro Benutzer oder Workspace dupliziert werden.
|
||||||
|
|
||||||
|
### Workspace-bezogene Daten
|
||||||
|
|
||||||
|
Spätere benutzerbezogene Beschaffungskontexte gehören zu einem Workspace:
|
||||||
|
|
||||||
|
- BuyingProject
|
||||||
|
- ProjectCandidate
|
||||||
|
- projektspezifische Profil-/Anforderungsauswahl
|
||||||
|
- Notification Preferences
|
||||||
|
- Beschaffungsdeadline
|
||||||
|
- Zielpreis
|
||||||
|
- Kaufentscheidungsstatus
|
||||||
|
- gegebenenfalls spätere Warenkorb-/Beschaffungsinformationen
|
||||||
|
|
||||||
|
Diese Objekte werden erst in späteren Slices implementiert.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. BuyingProject als zukünftiges Kernobjekt
|
||||||
|
|
||||||
|
Ein `BuyingProject` repräsentiert ein konkretes Beschaffungsvorhaben.
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
|
||||||
|
- Mobiler Entwicklungsrechner
|
||||||
|
- Lokaler LLM-Server
|
||||||
|
- Fernseher
|
||||||
|
- Kamera
|
||||||
|
|
||||||
|
Ein Workspace kann mehrere BuyingProjects parallel besitzen.
|
||||||
|
|
||||||
|
Ein BuyingProject kann mehrere ProjectCandidates beobachten.
|
||||||
|
|
||||||
|
```text
|
||||||
|
BuyingProject
|
||||||
|
├── Candidate A → ProductVariant
|
||||||
|
├── Candidate B → ProductVariant
|
||||||
|
└── Candidate C → ProductVariant
|
||||||
|
```
|
||||||
|
|
||||||
|
### Wichtig für v0.1
|
||||||
|
|
||||||
|
`BuyingProject` und `ProjectCandidate` müssen in v0.1 noch nicht implementiert werden.
|
||||||
|
|
||||||
|
Die aktuelle Implementierung darf jedoch nicht annehmen:
|
||||||
|
|
||||||
|
- es existiert genau ein aktives Profil,
|
||||||
|
- es existiert genau ein Kaufvorhaben,
|
||||||
|
- es gibt globale Singleton-Konfiguration für einen aktuellen Kauf,
|
||||||
|
- alle Daten gehören automatisch demselben Benutzer,
|
||||||
|
- es existiert nur eine gleichzeitig relevante Evaluation.
|
||||||
|
|
||||||
|
Mehrere unabhängige Profile und Evaluationen müssen bereits parallel koexistieren können.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Rolle von n8n
|
||||||
|
|
||||||
|
n8n bleibt eine externe Orchestrierungsschicht.
|
||||||
|
|
||||||
|
Spätere mögliche Aufgaben:
|
||||||
|
|
||||||
|
- Scheduler
|
||||||
|
- Preisabfragen
|
||||||
|
- Datenimport
|
||||||
|
- Benachrichtigungen
|
||||||
|
- externe Integrationen
|
||||||
|
- volatile oder häufig veränderte Workflows
|
||||||
|
|
||||||
|
Kern-Business-Logik bleibt im Programmcode, insbesondere wenn sie:
|
||||||
|
|
||||||
|
- deterministisch sein soll,
|
||||||
|
- versioniert werden muss,
|
||||||
|
- getestet werden muss,
|
||||||
|
- fachliche Entscheidungen reproduzierbar machen soll.
|
||||||
|
|
||||||
|
n8n darf für veränderliche Prozesslogik eingesetzt werden, aber nicht zur Voraussetzung für das Funktionieren der Buying Engine werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Zukünftiges Frontend
|
||||||
|
|
||||||
|
Das Zielsystem soll später eine einfach bedienbare Oberfläche erhalten.
|
||||||
|
|
||||||
|
Die Standardoberfläche soll auch für nichttechnische Familienmitglieder verständlich bleiben.
|
||||||
|
|
||||||
|
Die Engine darf gleichzeitig deutlich leistungsfähiger sein als die Oberfläche.
|
||||||
|
|
||||||
|
Voraussichtliche spätere Kernbereiche:
|
||||||
|
|
||||||
|
- Übersicht laufender Kaufprojekte
|
||||||
|
- Kandidaten je Projekt
|
||||||
|
- aktueller bester Deal
|
||||||
|
- Preisverlauf
|
||||||
|
- Produkt-Fit
|
||||||
|
- Offer-Fit
|
||||||
|
- DealScore
|
||||||
|
- verständliche Gründe
|
||||||
|
- Beobachtungsstatus
|
||||||
|
- konkrete Kaufempfehlung
|
||||||
|
- ggf. spätere Benachrichtigungs- und Warenkorbfunktionen
|
||||||
|
|
||||||
|
Keine dieser UI-Funktionen ist Bestandteil von v0.1.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Kaufentscheidungslogik
|
||||||
|
|
||||||
|
Langfristig soll das System nicht nur Scores liefern, sondern auch konfigurierbare Kaufentscheidungen unterstützen.
|
||||||
|
|
||||||
|
Beispiele möglicher Einflussfaktoren:
|
||||||
|
|
||||||
|
- Zielpreis
|
||||||
|
- Maximalpreis
|
||||||
|
- spätestes Beschaffungsdatum
|
||||||
|
- aktueller Preis
|
||||||
|
- Preisentwicklung der letzten Tage/Wochen/Monate
|
||||||
|
- Abstand zu historischem Median
|
||||||
|
- Produkt-Fit
|
||||||
|
- Offer-Fit
|
||||||
|
- Verfügbarkeit
|
||||||
|
- Lieferzeit
|
||||||
|
- Händlerqualität
|
||||||
|
- alternative Kandidaten
|
||||||
|
- Beschaffungsdringlichkeit
|
||||||
|
|
||||||
|
Die Kaufentscheidung soll nachvollziehbar und möglichst deterministisch bleiben.
|
||||||
|
|
||||||
|
LLMs dürfen später Erklärungen, Normalisierung oder qualitative Interpretation unterstützen, sollen aber keine nicht nachvollziehbare Blackbox-Entscheidung erzwingen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Datenquellen
|
||||||
|
|
||||||
|
Das Zielsystem muss mehrere Produkt- und Preisquellen unterstützen können.
|
||||||
|
|
||||||
|
Mögliche spätere Quellen:
|
||||||
|
|
||||||
|
- Amazon / Keepa
|
||||||
|
- Preisvergleichsportale
|
||||||
|
- Hersteller
|
||||||
|
- Händler
|
||||||
|
- weitere APIs
|
||||||
|
|
||||||
|
v0.1 implementiert weiterhin **keine reale externe Quelle**.
|
||||||
|
|
||||||
|
Die bestehende Source-Adapter-Grenze bleibt die vorgesehene Erweiterungsstelle.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Scope-Grenze für die laufende v0.1-Implementierung
|
||||||
|
|
||||||
|
Dieses Addendum darf nicht als Anlass dienen, v0.1 unnötig zu vergrößern.
|
||||||
|
|
||||||
|
Insbesondere **nicht jetzt implementieren**:
|
||||||
|
|
||||||
|
- echtes Frontend
|
||||||
|
- Authentifizierung
|
||||||
|
- User Management
|
||||||
|
- Workspace Management
|
||||||
|
- RBAC
|
||||||
|
- Einladungen
|
||||||
|
- BuyingProject
|
||||||
|
- ProjectCandidate
|
||||||
|
- Notifications
|
||||||
|
- n8n
|
||||||
|
- externe Preisquellen
|
||||||
|
- automatische Produktsuche
|
||||||
|
- Warenkorbmanagement
|
||||||
|
|
||||||
|
Ziel ist Architekturkompatibilität, nicht vorgezogene Feature-Entwicklung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. Arbeitsauftrag für die laufende Implementierung
|
||||||
|
|
||||||
|
Der Coding-Agent soll vor Fortsetzung des laufenden Builds einen kurzen Architecture-Alignment-Check durchführen.
|
||||||
|
|
||||||
|
### Prüfen
|
||||||
|
|
||||||
|
1. Entspricht die Repository-Struktur bereits der Monorepo-Struktur mit:
|
||||||
|
- `backend/`
|
||||||
|
- `frontend/`
|
||||||
|
- `docs/`
|
||||||
|
- Root-`docker-compose.yml`
|
||||||
|
|
||||||
|
2. Gibt es Annahmen über:
|
||||||
|
- genau einen Benutzer,
|
||||||
|
- genau ein aktives Profil,
|
||||||
|
- genau ein Kaufprojekt,
|
||||||
|
- globale Singleton-Zustände?
|
||||||
|
|
||||||
|
3. Sind Product, ProductVariant, Offer und PriceObservation weiterhin global und unabhängig von späteren Workspaces modelliert?
|
||||||
|
|
||||||
|
4. Ist das Backend klar von zukünftiger Frontend- und n8n-Logik getrennt?
|
||||||
|
|
||||||
|
### Änderungen
|
||||||
|
|
||||||
|
Nur notwendige strukturelle Anpassungen durchführen.
|
||||||
|
|
||||||
|
Keine vorgezogene Multi-User- oder Frontend-Funktionalität implementieren.
|
||||||
|
|
||||||
|
Die fachlichen Anforderungen aus `buying-agent-v0.1-spec-v1.1.md` bleiben vollständig gültig.
|
||||||
|
|
||||||
|
### Abschlussbericht
|
||||||
|
|
||||||
|
Dokumentieren:
|
||||||
|
|
||||||
|
- geänderte oder verschobene Dateien,
|
||||||
|
- notwendige Architekturänderungen,
|
||||||
|
- Änderungen an Datenbank/Migrationen,
|
||||||
|
- ausgeführte Tests,
|
||||||
|
- Testergebnisse,
|
||||||
|
- verbleibende Risiken für spätere Workspace-/BuyingProject-Erweiterung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13. Vorrang und Weiterentwicklung
|
||||||
|
|
||||||
|
Dieses Dokument ist ein Architektur-Addendum zur v0.1-Spezifikation.
|
||||||
|
|
||||||
|
Nach Abschluss von v0.1 sollen die Erkenntnisse aus:
|
||||||
|
|
||||||
|
- `buying-agent-v0.1-spec-v1.1.md`
|
||||||
|
- diesem Addendum
|
||||||
|
- der realen Implementierung
|
||||||
|
|
||||||
|
in eine konsolidierte Folgespezifikation für den nächsten Slice übernommen werden.
|
||||||
|
|
||||||
|
Die Kernregel lautet:
|
||||||
|
|
||||||
|
> Nicht vorzeitig bauen, aber bekannte Zielanforderungen nicht architektonisch verbauen.
|
||||||
987
.specs/buying-agent-frontend-mvp-spec-v0.1.md
Normal file
987
.specs/buying-agent-frontend-mvp-spec-v0.1.md
Normal file
|
|
@ -0,0 +1,987 @@
|
||||||
|
# Buying Agent – Frontend MVP Specification v0.1
|
||||||
|
|
||||||
|
**Status:** implementierungsbereit
|
||||||
|
**Spezifikationsversion:** 0.1
|
||||||
|
**Stand:** 2026-10-04
|
||||||
|
|
||||||
|
**Ergänzt verbindlich:**
|
||||||
|
1. `buying-agent-v0.1-spec-v1.1.md`
|
||||||
|
2. `buying-agent-architecture-addendum-v1.md`
|
||||||
|
3. `buying-agent-source-discovery-spec-v0.1.md`
|
||||||
|
|
||||||
|
Diese Frontend-Spezifikation hebt für den nutzbaren MVP die frühere Aussage „kein Frontend in diesem Slice“ auf.
|
||||||
|
Sie ändert keine fachliche Backend-Logik; sie definiert die Benutzeroberfläche und die dafür notwendigen API-Verträge.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Ziel
|
||||||
|
|
||||||
|
Der Buying Agent soll als **nutzbare PWA** einen vollständigen vertikalen Ablauf unterstützen:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BuyingProject anlegen
|
||||||
|
↓
|
||||||
|
Produktkategorie wählen
|
||||||
|
↓
|
||||||
|
dynamische Kriterien aus Source-Taxonomie auswählen
|
||||||
|
↓
|
||||||
|
Kriterien konfigurieren
|
||||||
|
↓
|
||||||
|
Geizhals-Suche generieren
|
||||||
|
↓
|
||||||
|
SearchRun starten / Status verfolgen
|
||||||
|
↓
|
||||||
|
Kandidaten sehen
|
||||||
|
↓
|
||||||
|
Ranking + Shortlist sehen
|
||||||
|
↓
|
||||||
|
Kandidat öffnen
|
||||||
|
↓
|
||||||
|
Status, Historie, bisherige Ergebnisse und Empfehlungen sehen
|
||||||
|
↓
|
||||||
|
Kriterien bei Bedarf anpassen
|
||||||
|
```
|
||||||
|
|
||||||
|
Der MVP muss diesen Flow über die Oberfläche vollständig nutzbar machen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Technologiestack
|
||||||
|
|
||||||
|
Frontend:
|
||||||
|
|
||||||
|
- React
|
||||||
|
- TypeScript
|
||||||
|
- Vite
|
||||||
|
- PWA
|
||||||
|
- TanStack Query für Server-State / Data Fetching
|
||||||
|
- React Router oder äquivalente schlanke Routing-Lösung
|
||||||
|
- schlanke komponentenbasierte UI
|
||||||
|
- kein Next.js
|
||||||
|
- kein SSR
|
||||||
|
- keine SEO-Anforderungen
|
||||||
|
|
||||||
|
Das Frontend liegt im bestehenden Monorepo:
|
||||||
|
|
||||||
|
```text
|
||||||
|
buying-agent/
|
||||||
|
├── backend/
|
||||||
|
├── frontend/
|
||||||
|
├── docs/
|
||||||
|
└── docker-compose.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Backend bleibt System of Record.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Gestaltungsprinzipien
|
||||||
|
|
||||||
|
1. **Einfach für Standardnutzer, leistungsfähig im Hintergrund.**
|
||||||
|
2. Keine technische Datenbank-/Engine-Sicht in der Standardoberfläche.
|
||||||
|
3. Dynamische Kriterien statt fest verdrahteter Notebook-Felder.
|
||||||
|
4. Responsive von Anfang an.
|
||||||
|
5. Zwei bewusst gestaltete Hauptlayouts:
|
||||||
|
- Desktop
|
||||||
|
- Mobile
|
||||||
|
6. Nüchternes, klares Design.
|
||||||
|
7. Bedienbarkeit vor visueller Komplexität.
|
||||||
|
8. Admin-/Source-Themen von der normalen Nutzung trennen.
|
||||||
|
9. Keine Businesslogik im Frontend.
|
||||||
|
10. Frontend konsumiert ausschließlich die versionierte Backend-API.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. PWA und Responsive Layout
|
||||||
|
|
||||||
|
### Desktop
|
||||||
|
|
||||||
|
Desktop verwendet primär:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Sidebar
|
||||||
|
+ Hauptinhalt
|
||||||
|
+ optionaler Detailbereich
|
||||||
|
```
|
||||||
|
|
||||||
|
Beispiel:
|
||||||
|
|
||||||
|
```text
|
||||||
|
┌───────────────┬──────────────────────────────────────┐
|
||||||
|
│ Buying Agent │ Projekt: Mobiler Entwicklungsrechner│
|
||||||
|
│ │ │
|
||||||
|
│ Projekte │ Status / Suche / Ranking / Shortlist│
|
||||||
|
│ Admin │ │
|
||||||
|
│ Einstellungen │ │
|
||||||
|
└───────────────┴──────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### Mobile
|
||||||
|
|
||||||
|
Mobile verwendet primär:
|
||||||
|
|
||||||
|
- Bottom Navigation oder kompakte Hauptnavigation
|
||||||
|
- Vollbild-Detailseiten
|
||||||
|
- Tabs innerhalb eines BuyingProjects
|
||||||
|
- keine verkleinerte Desktop-Sidebar
|
||||||
|
|
||||||
|
Beispiel:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Projekt
|
||||||
|
────────────────
|
||||||
|
Übersicht
|
||||||
|
Kandidaten
|
||||||
|
Shortlist
|
||||||
|
Kriterien
|
||||||
|
Historie
|
||||||
|
|
||||||
|
[ Projekte ] [ Suche ] [ Admin ]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Mitai-Bezug
|
||||||
|
|
||||||
|
Falls im bestehenden Mitai-Code wiederverwendbare Layout-, PWA-, Responsive- oder Navigationsmuster vorhanden sind, sollen diese bevorzugt übernommen oder konsistent nachgebildet werden.
|
||||||
|
|
||||||
|
Keine fachliche Kopplung an Mitai.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Hauptnavigation
|
||||||
|
|
||||||
|
Der MVP benötigt mindestens:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Projekte
|
||||||
|
Admin
|
||||||
|
```
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Systemstatus
|
||||||
|
```
|
||||||
|
|
||||||
|
Noch nicht erforderlich:
|
||||||
|
|
||||||
|
- Benutzerverwaltung
|
||||||
|
- Warenkorb
|
||||||
|
- Benachrichtigungseinstellungen
|
||||||
|
- Account-/Profilverwaltung
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Screen 1 – BuyingProject-Liste
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/projects
|
||||||
|
```
|
||||||
|
|
||||||
|
Funktionen:
|
||||||
|
|
||||||
|
- aktive BuyingProjects anzeigen
|
||||||
|
- Projektname
|
||||||
|
- Kategorie
|
||||||
|
- Status
|
||||||
|
- letzter erfolgreicher SearchRun
|
||||||
|
- Anzahl Kandidaten
|
||||||
|
- Anzahl Shortlist-Kandidaten
|
||||||
|
- aktueller Discovery-Status
|
||||||
|
- Button „Neues Projekt“
|
||||||
|
|
||||||
|
Mögliche Status:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ACTIVE
|
||||||
|
PURCHASED
|
||||||
|
ARCHIVED
|
||||||
|
OUT_OF_SYNC
|
||||||
|
BLOCKED
|
||||||
|
TOO_BROAD
|
||||||
|
```
|
||||||
|
|
||||||
|
Die Liste darf keine produktkategoriespezifischen Felder voraussetzen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Screen 2 – BuyingProject anlegen
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/projects/new
|
||||||
|
```
|
||||||
|
|
||||||
|
### Schritt 1 – Basisdaten
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
```text
|
||||||
|
name
|
||||||
|
source
|
||||||
|
source_category
|
||||||
|
market
|
||||||
|
currency
|
||||||
|
shortlist_size
|
||||||
|
schedule_interval
|
||||||
|
```
|
||||||
|
|
||||||
|
Default:
|
||||||
|
|
||||||
|
```text
|
||||||
|
shortlist_size = 5
|
||||||
|
schedule_interval_minutes = 180
|
||||||
|
```
|
||||||
|
|
||||||
|
### Schritt 2 – Kriterien
|
||||||
|
|
||||||
|
Nach Auswahl der Source-Kategorie lädt das Frontend dynamisch:
|
||||||
|
|
||||||
|
- Canonical Criteria
|
||||||
|
- Source Filter Definitions
|
||||||
|
- CriterionMappings
|
||||||
|
- Mapping-Status
|
||||||
|
|
||||||
|
Das Frontend erzeugt daraus dynamisch die Eingabefelder.
|
||||||
|
|
||||||
|
Es gibt **keine fest codierten Notebook-Felder**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Dynamische Kriterien
|
||||||
|
|
||||||
|
Verfügbare Kriterien entstehen aus:
|
||||||
|
|
||||||
|
```text
|
||||||
|
SourceCategory
|
||||||
|
↓
|
||||||
|
SourceFilterDefinitions
|
||||||
|
↓
|
||||||
|
CriterionDefinitions
|
||||||
|
↓
|
||||||
|
CriterionMappings
|
||||||
|
```
|
||||||
|
|
||||||
|
Das Frontend rendert Controls anhand des Typs.
|
||||||
|
|
||||||
|
### Unterstützte Control-Typen
|
||||||
|
|
||||||
|
```text
|
||||||
|
BOOLEAN
|
||||||
|
ENUM
|
||||||
|
MULTI_ENUM
|
||||||
|
NUMERIC_RANGE
|
||||||
|
NUMERIC_THRESHOLD
|
||||||
|
TEXT
|
||||||
|
```
|
||||||
|
|
||||||
|
Für ein Kriterium sollen mindestens konfigurierbar sein:
|
||||||
|
|
||||||
|
```text
|
||||||
|
operator
|
||||||
|
value
|
||||||
|
mandatory / optional
|
||||||
|
```
|
||||||
|
|
||||||
|
Wenn vom Scoring unterstützt zusätzlich:
|
||||||
|
|
||||||
|
```text
|
||||||
|
weight
|
||||||
|
preference strategy
|
||||||
|
acceptable
|
||||||
|
ideal
|
||||||
|
direction
|
||||||
|
```
|
||||||
|
|
||||||
|
### Sichtbarer Mapping-Status
|
||||||
|
|
||||||
|
Pro Kriterium:
|
||||||
|
|
||||||
|
```text
|
||||||
|
VERIFIED
|
||||||
|
LEARNED_UNVERIFIED
|
||||||
|
INTERNAL_ONLY
|
||||||
|
BROKEN
|
||||||
|
DISABLED
|
||||||
|
```
|
||||||
|
|
||||||
|
Standardnutzer sollen primär verständliche Hinweise sehen:
|
||||||
|
|
||||||
|
- „Wird direkt bei Geizhals gefiltert“
|
||||||
|
- „Wird erst intern bewertet“
|
||||||
|
- „Filter derzeit nicht verfügbar“
|
||||||
|
|
||||||
|
Technische IDs bleiben im Admin-Bereich.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Kriterienänderungen
|
||||||
|
|
||||||
|
Wenn Kriterien geändert werden:
|
||||||
|
|
||||||
|
```text
|
||||||
|
criteria_version++
|
||||||
|
```
|
||||||
|
|
||||||
|
Die bestehende DiscoverySearch wird:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OUT_OF_SYNC
|
||||||
|
```
|
||||||
|
|
||||||
|
Danach muss das Backend eine neue Query aus den aktuellen:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Criteria
|
||||||
|
+
|
||||||
|
VERIFIED Mappings
|
||||||
|
```
|
||||||
|
|
||||||
|
erzeugen.
|
||||||
|
|
||||||
|
Das Frontend zeigt klar:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Suchkriterien wurden geändert.
|
||||||
|
Die externe Suche muss aktualisiert werden.
|
||||||
|
```
|
||||||
|
|
||||||
|
Aktion:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Suche aktualisieren
|
||||||
|
```
|
||||||
|
|
||||||
|
Danach:
|
||||||
|
|
||||||
|
```text
|
||||||
|
generate-query
|
||||||
|
→ optional run
|
||||||
|
→ neuer SearchRun
|
||||||
|
→ neues Ranking
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Screen 3 – BuyingProject-Detail
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/projects/:projectId
|
||||||
|
```
|
||||||
|
|
||||||
|
Desktop: strukturierte Detailansicht.
|
||||||
|
|
||||||
|
Mobile: Tabs.
|
||||||
|
|
||||||
|
Mindestens folgende Bereiche:
|
||||||
|
|
||||||
|
### Übersicht
|
||||||
|
|
||||||
|
- Projektstatus
|
||||||
|
- Kategorie
|
||||||
|
- letzter SearchRun
|
||||||
|
- Discovery-Status
|
||||||
|
- Trefferzahl
|
||||||
|
- Kandidatenzahl
|
||||||
|
- Shortlist-Größe
|
||||||
|
- Zeitpunkt nächste geplante Suche
|
||||||
|
- wichtige Warnungen
|
||||||
|
|
||||||
|
### Kandidaten
|
||||||
|
|
||||||
|
- aktuelle Kandidaten
|
||||||
|
- ProductScore
|
||||||
|
- product_eligibility
|
||||||
|
- aktueller best_price, sofern vorhanden
|
||||||
|
- Diff-Status
|
||||||
|
- Lifecycle-Status
|
||||||
|
- Shortlist-Markierung
|
||||||
|
|
||||||
|
### Shortlist
|
||||||
|
|
||||||
|
Top-N gemäß Projektkonfiguration.
|
||||||
|
|
||||||
|
### Kriterien
|
||||||
|
|
||||||
|
Dynamische Kriterien anzeigen und bearbeiten.
|
||||||
|
|
||||||
|
### Historie
|
||||||
|
|
||||||
|
- SearchRuns
|
||||||
|
- wichtige Statusänderungen
|
||||||
|
- bisherige Rankings
|
||||||
|
- soweit vorhanden frühere Bewertungen / Empfehlungen
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Screen 4 – Kandidatenliste
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/projects/:projectId/candidates
|
||||||
|
```
|
||||||
|
|
||||||
|
Darstellung:
|
||||||
|
|
||||||
|
- sortierbare Liste/Tabelle auf Desktop
|
||||||
|
- Cards/List Items auf Mobile
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Produktname
|
||||||
|
Hersteller
|
||||||
|
wichtige dynamische Attribute
|
||||||
|
best_price
|
||||||
|
ProductScore
|
||||||
|
product_eligibility
|
||||||
|
diff_status
|
||||||
|
lifecycle_status
|
||||||
|
shortlist_status
|
||||||
|
last_seen
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dynamische Attribute
|
||||||
|
|
||||||
|
Welche Eigenschaften sichtbar sind, darf nicht notebook-spezifisch hart codiert sein.
|
||||||
|
|
||||||
|
Mögliche Strategie:
|
||||||
|
|
||||||
|
1. Source-/Category-Definition liefert relevante display attributes.
|
||||||
|
2. Projekt kann später bevorzugte sichtbare Attribute konfigurieren.
|
||||||
|
3. MVP darf zunächst die wichtigsten vorhandenen normalisierten Attribute anzeigen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. Screen 5 – Kandidatendetail
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/projects/:projectId/candidates/:candidateId
|
||||||
|
```
|
||||||
|
|
||||||
|
Der Nutzer soll in ein Produkt „hineinspringen“ können.
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
### Aktueller Status
|
||||||
|
|
||||||
|
- Name
|
||||||
|
- Hersteller
|
||||||
|
- Source-Link
|
||||||
|
- ProductScore
|
||||||
|
- product_eligibility
|
||||||
|
- Shortlist-Status
|
||||||
|
- aktueller best_price
|
||||||
|
- letzte Sichtung
|
||||||
|
|
||||||
|
### Attribute
|
||||||
|
|
||||||
|
Alle verfügbaren normalisierten dynamischen Produktattribute.
|
||||||
|
|
||||||
|
Keine notebook-spezifische Feldliste.
|
||||||
|
|
||||||
|
### Historie
|
||||||
|
|
||||||
|
- Preisentwicklung aus ProductObservations
|
||||||
|
- relevante Attributänderungen
|
||||||
|
- NEW / CHANGED / MISSING / DISAPPEARED
|
||||||
|
- SearchRun-Bezug
|
||||||
|
|
||||||
|
### Bewertungen / Ergebnisse
|
||||||
|
|
||||||
|
- aktuelle Bewertung
|
||||||
|
- frühere gespeicherte Bewertungen, soweit Backend vorhanden
|
||||||
|
- Gründe / Constraint-Ergebnisse
|
||||||
|
- Coverage / fehlende Daten
|
||||||
|
|
||||||
|
### Spätere Erweiterbarkeit
|
||||||
|
|
||||||
|
Dieser Screen wird später Zielort für:
|
||||||
|
|
||||||
|
- Deep Enrichment
|
||||||
|
- Tests
|
||||||
|
- Reviews
|
||||||
|
- Benchmarks
|
||||||
|
- Händlervergleich
|
||||||
|
- Kaufempfehlung
|
||||||
|
|
||||||
|
Diese Funktionen sind noch nicht Bestandteil dieses MVP-Slices.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13. Screen 6 – Shortlist
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/projects/:projectId/shortlist
|
||||||
|
```
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
|
||||||
|
Nur die aktuell relevantesten Produkte zeigen.
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
- Rang
|
||||||
|
- Produkt
|
||||||
|
- ProductScore
|
||||||
|
- Product Eligibility
|
||||||
|
- aktueller Preis
|
||||||
|
- relevante Stärken
|
||||||
|
- relevante Schwächen / fehlende Daten
|
||||||
|
- letzter Änderungsstatus
|
||||||
|
|
||||||
|
Default:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Top 5
|
||||||
|
```
|
||||||
|
|
||||||
|
konfigurierbar am BuyingProject.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 14. Screen 7 – Search Runs / Historie
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/projects/:projectId/history
|
||||||
|
```
|
||||||
|
|
||||||
|
Anzeige:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Zeitpunkt
|
||||||
|
Status
|
||||||
|
Trefferzahl
|
||||||
|
Page Count
|
||||||
|
NEW
|
||||||
|
CHANGED
|
||||||
|
MISSING
|
||||||
|
DISAPPEARED
|
||||||
|
TOO_BROAD
|
||||||
|
Fehler / BLOCKED
|
||||||
|
```
|
||||||
|
|
||||||
|
Ein SearchRun muss geöffnet werden können, um Detailinformationen zu sehen.
|
||||||
|
|
||||||
|
Kein HTML anzeigen oder archivieren.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 15. Screen 8 – Admin: Taxonomie & Mapping
|
||||||
|
|
||||||
|
Route:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/admin/sources
|
||||||
|
/admin/sources/:source/categories/:category
|
||||||
|
```
|
||||||
|
|
||||||
|
Dieser Bereich darf technischer sein.
|
||||||
|
|
||||||
|
Funktionen:
|
||||||
|
|
||||||
|
- gelernte Kategorien sehen
|
||||||
|
- SourceFilterDefinitions sehen
|
||||||
|
- SourceFilterOptions/Ranges sehen
|
||||||
|
- Canonical Criteria sehen
|
||||||
|
- CriterionMappings sehen
|
||||||
|
- Mapping-Status ändern
|
||||||
|
- fehlerhafte kleine Mapping-Änderungen direkt korrigieren
|
||||||
|
- BROKEN / DISABLED erkennen
|
||||||
|
- Taxonomie neu einlesen
|
||||||
|
- Query-Mapping nachvollziehen
|
||||||
|
|
||||||
|
### Wichtig
|
||||||
|
|
||||||
|
Das Frontend darf kein URL-Encoding selbst erfinden.
|
||||||
|
|
||||||
|
Es editiert/pflegt nur persistierte Mapping-Daten.
|
||||||
|
|
||||||
|
Backend bleibt für Validierung und Query-Generation verantwortlich.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 16. Server State und Data Fetching
|
||||||
|
|
||||||
|
Empfehlung:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TanStack Query
|
||||||
|
```
|
||||||
|
|
||||||
|
für:
|
||||||
|
|
||||||
|
- Projekte
|
||||||
|
- Kriterien
|
||||||
|
- Taxonomie
|
||||||
|
- Mappings
|
||||||
|
- SearchRuns
|
||||||
|
- Kandidaten
|
||||||
|
- Shortlist
|
||||||
|
- Produktdetails
|
||||||
|
|
||||||
|
Client-State nur für:
|
||||||
|
|
||||||
|
- UI-Zustände
|
||||||
|
- Formulare
|
||||||
|
- Tabs
|
||||||
|
- lokale Filter
|
||||||
|
|
||||||
|
Keine fachlich relevante Server-Wahrheit ausschließlich im Browser halten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 17. API-Verträge
|
||||||
|
|
||||||
|
Das Frontend nutzt mindestens:
|
||||||
|
|
||||||
|
### Projekte
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/buying-projects
|
||||||
|
POST /api/v1/buying-projects
|
||||||
|
GET /api/v1/buying-projects/{id}
|
||||||
|
PATCH /api/v1/buying-projects/{id}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Projektkriterien
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/buying-projects/{id}/criteria
|
||||||
|
PUT /api/v1/buying-projects/{id}/criteria
|
||||||
|
```
|
||||||
|
|
||||||
|
Falls Backend intern versionierte Ressourcen bevorzugt, kann PUT eine neue Version erzeugen.
|
||||||
|
|
||||||
|
### Taxonomie
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /api/v1/sources/{source}/categories/{category}/discover-taxonomy
|
||||||
|
GET /api/v1/sources/{source}/categories/{category}/filters
|
||||||
|
```
|
||||||
|
|
||||||
|
### Criterion Definitions / Mappings
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/criterion-definitions
|
||||||
|
GET /api/v1/criterion-mappings
|
||||||
|
POST /api/v1/criterion-mappings
|
||||||
|
PATCH /api/v1/criterion-mappings/{id}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Discovery Search
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/buying-projects/{id}/discovery-search
|
||||||
|
POST /api/v1/discovery-searches/{id}/generate-query
|
||||||
|
POST /api/v1/discovery-searches/{id}/run
|
||||||
|
```
|
||||||
|
|
||||||
|
### Search Runs
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/discovery-searches/{id}/runs
|
||||||
|
GET /api/v1/search-runs/{id}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Candidates
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/buying-projects/{id}/candidates
|
||||||
|
GET /api/v1/buying-projects/{id}/shortlist
|
||||||
|
GET /api/v1/buying-projects/{id}/candidates/{candidateId}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 18. Status- und Fehlerdarstellung
|
||||||
|
|
||||||
|
Frontend muss mindestens folgende Zustände sichtbar machen:
|
||||||
|
|
||||||
|
### Discovery
|
||||||
|
|
||||||
|
```text
|
||||||
|
SUCCESS
|
||||||
|
PARTIAL
|
||||||
|
FAILED
|
||||||
|
BLOCKED
|
||||||
|
TOO_BROAD
|
||||||
|
OUT_OF_SYNC
|
||||||
|
```
|
||||||
|
|
||||||
|
### Candidate Diff
|
||||||
|
|
||||||
|
```text
|
||||||
|
NEW
|
||||||
|
CHANGED
|
||||||
|
UNCHANGED
|
||||||
|
MISSING
|
||||||
|
DISAPPEARED
|
||||||
|
```
|
||||||
|
|
||||||
|
### Matching
|
||||||
|
|
||||||
|
```text
|
||||||
|
MATCHED
|
||||||
|
MATCH_UNCERTAIN
|
||||||
|
```
|
||||||
|
|
||||||
|
### Mapping
|
||||||
|
|
||||||
|
```text
|
||||||
|
DISCOVERED
|
||||||
|
LEARNED_UNVERIFIED
|
||||||
|
VERIFIED
|
||||||
|
BROKEN
|
||||||
|
DISABLED
|
||||||
|
INTERNAL_ONLY
|
||||||
|
```
|
||||||
|
|
||||||
|
Fehlertexte für Standardnutzer verständlich formulieren.
|
||||||
|
|
||||||
|
Technische Details optional aufklappbar.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 19. Loading / Empty / Error States
|
||||||
|
|
||||||
|
Jeder Screen braucht explizit:
|
||||||
|
|
||||||
|
- Loading
|
||||||
|
- Empty
|
||||||
|
- Error
|
||||||
|
- Retry
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Noch keine Kandidaten gefunden.
|
||||||
|
Noch kein SearchRun ausgeführt.
|
||||||
|
Taxonomie wurde noch nicht geladen.
|
||||||
|
Diese Suche ist derzeit BLOCKED.
|
||||||
|
Die Kriterien wurden geändert; Suche ist OUT_OF_SYNC.
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 20. PWA-Grundgerüst
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
- installierbare PWA
|
||||||
|
- Manifest
|
||||||
|
- Icons / App-Metadaten
|
||||||
|
- responsive shell
|
||||||
|
- app-like navigation
|
||||||
|
- sinnvolle Offline-Fallback-Seite
|
||||||
|
|
||||||
|
Keine Offline-Businesslogik erforderlich.
|
||||||
|
|
||||||
|
Keine komplexe Synchronisationslogik in v0.1.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 21. Tests
|
||||||
|
|
||||||
|
### Unit / Component
|
||||||
|
|
||||||
|
- dynamischer Criterion Renderer je Filtertyp
|
||||||
|
- Mapping-Status-Darstellung
|
||||||
|
- Candidate Status-Darstellung
|
||||||
|
- responsive Komponenten
|
||||||
|
- Formvalidierung
|
||||||
|
|
||||||
|
### API Integration
|
||||||
|
|
||||||
|
- Projektliste
|
||||||
|
- Projektanlage
|
||||||
|
- Kriterien laden/speichern
|
||||||
|
- SearchRun starten
|
||||||
|
- Kandidaten laden
|
||||||
|
- Shortlist laden
|
||||||
|
- Admin-Mapping aktualisieren
|
||||||
|
|
||||||
|
### E2E
|
||||||
|
|
||||||
|
Mindestens ein vollständiger Notebook-Pilot:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Projekt anlegen
|
||||||
|
→ Kategorie Notebooks
|
||||||
|
→ dynamische Kriterien laden
|
||||||
|
→ Kriterien setzen
|
||||||
|
→ Geizhals Query generieren
|
||||||
|
→ SearchRun starten
|
||||||
|
→ Kandidaten erscheinen
|
||||||
|
→ Ranking sichtbar
|
||||||
|
→ Shortlist sichtbar
|
||||||
|
→ Kandidat öffnen
|
||||||
|
→ Historie sichtbar
|
||||||
|
→ Kriterium ändern
|
||||||
|
→ OUT_OF_SYNC sichtbar
|
||||||
|
→ Query neu erzeugen
|
||||||
|
```
|
||||||
|
|
||||||
|
### Responsive E2E
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
- Desktop-Viewport
|
||||||
|
- Mobile-Viewport
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 22. Nicht implementieren
|
||||||
|
|
||||||
|
In diesem Slice weiterhin nicht:
|
||||||
|
|
||||||
|
- Login/Auth
|
||||||
|
- RBAC
|
||||||
|
- Einladungen
|
||||||
|
- Multi-User-UI
|
||||||
|
- Keepa
|
||||||
|
- Idealo
|
||||||
|
- WhatsApp
|
||||||
|
- Warenkorb
|
||||||
|
- Kaufautomatisierung
|
||||||
|
- LLM-Automation
|
||||||
|
- Deep Enrichment
|
||||||
|
- finale Decision Engine
|
||||||
|
|
||||||
|
Die Architektur darf diese Erweiterungen nicht verbauen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 23. Definition of Done
|
||||||
|
|
||||||
|
- [ ] React + TypeScript + Vite Frontend existiert im Monorepo.
|
||||||
|
- [ ] PWA ist installierbar.
|
||||||
|
- [ ] Desktop- und Mobile-Layouts sind bewusst getrennt gestaltet.
|
||||||
|
- [ ] BuyingProjects können über die UI angelegt und geöffnet werden.
|
||||||
|
- [ ] Produktkategorie kann ausgewählt werden.
|
||||||
|
- [ ] Kriterien werden dynamisch aus Taxonomie / CriterionDefinitions erzeugt.
|
||||||
|
- [ ] Keine notebook-spezifischen Criteria-Felder im Frontend-Core.
|
||||||
|
- [ ] Kriterien können bearbeitet und gespeichert werden.
|
||||||
|
- [ ] Mapping-Status ist sichtbar.
|
||||||
|
- [ ] INTERNAL_ONLY-Kriterien können gespeichert werden.
|
||||||
|
- [ ] Discovery Search kann generiert werden.
|
||||||
|
- [ ] SearchRun kann aus der UI gestartet werden.
|
||||||
|
- [ ] SearchRun-Status ist sichtbar.
|
||||||
|
- [ ] Kandidatenliste funktioniert.
|
||||||
|
- [ ] Ranking ist sichtbar.
|
||||||
|
- [ ] Shortlist funktioniert.
|
||||||
|
- [ ] Kandidatendetail zeigt dynamische Attribute und Historie.
|
||||||
|
- [ ] Projekt-Historie zeigt SearchRuns und relevante Änderungen.
|
||||||
|
- [ ] Kriterienänderung setzt/zeigt OUT_OF_SYNC und ermöglicht Regeneration.
|
||||||
|
- [ ] Admin-Bereich erlaubt Prüfung/Korrektur von Source-Taxonomie und Mappings.
|
||||||
|
- [ ] Loading-/Empty-/Error-Zustände existieren.
|
||||||
|
- [ ] E2E-Test des vollständigen Notebook-Piloten ist grün.
|
||||||
|
- [ ] Desktop- und Mobile-E2E sind grün.
|
||||||
|
- [ ] Backend bleibt alleinige Quelle fachlicher Wahrheit.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Coding-Agent-Arbeitsauftrag
|
||||||
|
|
||||||
|
Verwende als verbindliche Anforderungen:
|
||||||
|
|
||||||
|
1. `buying-agent-v0.1-spec-v1.1.md`
|
||||||
|
2. `buying-agent-architecture-addendum-v1.md`
|
||||||
|
3. `buying-agent-source-discovery-spec-v0.1.md`
|
||||||
|
4. `buying-agent-frontend-mvp-spec-v0.1.md`
|
||||||
|
|
||||||
|
Die Frontend-MVP-Spec hebt für den nutzbaren MVP die frühere Scope-Aussage „kein Frontend“ auf.
|
||||||
|
|
||||||
|
## 1. Vor Implementierung
|
||||||
|
|
||||||
|
Prüfe den aktuellen Repository-Stand gegen alle vier Dokumente.
|
||||||
|
|
||||||
|
Berichte kurz:
|
||||||
|
|
||||||
|
- welche Backend-APIs für den Frontend-Flow bereits existieren,
|
||||||
|
- welche Endpunkte fehlen,
|
||||||
|
- ob aktuelle DTOs dynamische Kriterien und Produktattribute vollständig tragen,
|
||||||
|
- ob bestehender Code notebook-spezifische Felder enthält,
|
||||||
|
- welche Änderungen rein additiv möglich sind.
|
||||||
|
|
||||||
|
Keine größere strukturelle Abweichung ohne Rückfrage.
|
||||||
|
|
||||||
|
## 2. Vertikalen MVP implementieren
|
||||||
|
|
||||||
|
Nicht Backend und Frontend getrennt in langen Phasen bauen.
|
||||||
|
|
||||||
|
Stattdessen vertikal:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Projektliste
|
||||||
|
→ Projektanlage
|
||||||
|
→ dynamische Kriterien
|
||||||
|
→ Query Generation
|
||||||
|
→ SearchRun
|
||||||
|
→ Kandidaten
|
||||||
|
→ Ranking
|
||||||
|
→ Shortlist
|
||||||
|
→ Kandidatendetail
|
||||||
|
```
|
||||||
|
|
||||||
|
Jeder Schritt soll nach Implementierung unmittelbar über die UI prüfbar sein.
|
||||||
|
|
||||||
|
## 3. Dynamik sicherstellen
|
||||||
|
|
||||||
|
Keine festen Felder wie:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ram_gb
|
||||||
|
storage_gb
|
||||||
|
weight_kg
|
||||||
|
display_inches
|
||||||
|
```
|
||||||
|
|
||||||
|
in generischen Frontend-Komponenten hardcoden.
|
||||||
|
|
||||||
|
Controls und sichtbare Produktattribute werden metadata-driven erzeugt.
|
||||||
|
|
||||||
|
Notebook-spezifische Werte dürfen nur als gelernte/gespeicherte Daten vorkommen.
|
||||||
|
|
||||||
|
## 4. UI
|
||||||
|
|
||||||
|
- React + TypeScript + Vite
|
||||||
|
- PWA
|
||||||
|
- TanStack Query
|
||||||
|
- einfache, nüchterne Bedienung
|
||||||
|
- Desktop: Sidebar + Content/Detail
|
||||||
|
- Mobile: kompakte Navigation / Bottom Navigation + Vollbild-Details
|
||||||
|
- vorhandene Mitai-Muster wiederverwenden, falls im zugänglichen Repo vorhanden und passend
|
||||||
|
|
||||||
|
## 5. Tests
|
||||||
|
|
||||||
|
Führe aus:
|
||||||
|
|
||||||
|
- Frontend Unit/Component Tests
|
||||||
|
- API Integration Tests
|
||||||
|
- E2E vollständiger Notebook-Pilot
|
||||||
|
- Desktop E2E
|
||||||
|
- Mobile E2E
|
||||||
|
|
||||||
|
## 6. Abschlussbericht
|
||||||
|
|
||||||
|
Dokumentiere:
|
||||||
|
|
||||||
|
- implementierte Screens
|
||||||
|
- neue/angepasste Backend-Endpunkte
|
||||||
|
- verwendete Frontend-Bibliotheken
|
||||||
|
- responsive Verhalten
|
||||||
|
- ausgeführte Tests und Ergebnisse
|
||||||
|
- offene Abweichungen
|
||||||
|
- Risiken für spätere Multi-User-, Deep-Enrichment- und Decision-Erweiterungen
|
||||||
1329
.specs/buying-agent-frontend-mvp-spec-v0.2.md
Normal file
1329
.specs/buying-agent-frontend-mvp-spec-v0.2.md
Normal file
File diff suppressed because it is too large
Load Diff
1708
.specs/buying-agent-frontend-mvp-spec-v0.3.md
Normal file
1708
.specs/buying-agent-frontend-mvp-spec-v0.3.md
Normal file
File diff suppressed because it is too large
Load Diff
986
.specs/buying-agent-source-discovery-spec-v0.1.md
Normal file
986
.specs/buying-agent-source-discovery-spec-v0.1.md
Normal file
|
|
@ -0,0 +1,986 @@
|
||||||
|
# Buying Agent – Source & Discovery Specification v0.1
|
||||||
|
|
||||||
|
**Status:** implementierungsbereit
|
||||||
|
**Spezifikationsversion:** 0.1
|
||||||
|
**Stand:** 2026-10-04
|
||||||
|
**Ergänzt:** `buying-agent-v0.1-spec-v1.1.md` und `buying-agent-architecture-addendum-v1.md`
|
||||||
|
|
||||||
|
## 1. Zweck
|
||||||
|
|
||||||
|
Diese Spezifikation definiert den ersten real nutzbaren Discovery-Slice des Buying Agents.
|
||||||
|
|
||||||
|
Ziel ist, externe Produktsuchen automatisiert aus den Kriterien eines BuyingProjects abzuleiten, regelmäßig auszuführen, Ergebnisse dauerhaft zu verwalten, Veränderungen zu erkennen und die Treffer anschließend durch die eigene Buying-Engine zu bewerten.
|
||||||
|
|
||||||
|
Der erste Pilot erfolgt mit **Geizhals, Kategorie Notebooks**. Die Architektur darf jedoch nicht notebook-spezifisch sein.
|
||||||
|
|
||||||
|
```text
|
||||||
|
BuyingProject
|
||||||
|
↓
|
||||||
|
Canonical Criteria
|
||||||
|
↓
|
||||||
|
Source Taxonomy + Mapping
|
||||||
|
↓
|
||||||
|
generierte Geizhals-Suche
|
||||||
|
↓
|
||||||
|
periodischer SearchRun
|
||||||
|
↓
|
||||||
|
persistierte Produkte + Beobachtungen
|
||||||
|
↓
|
||||||
|
Diff zum vorherigen Lauf
|
||||||
|
↓
|
||||||
|
NEW / CHANGED / MISSING / DISAPPEARED
|
||||||
|
↓
|
||||||
|
interne Constraints + Ranking
|
||||||
|
↓
|
||||||
|
Shortlist
|
||||||
|
↓
|
||||||
|
später Deep Enrichment
|
||||||
|
↓
|
||||||
|
später Decision
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. Fachliche Leitprinzipien
|
||||||
|
|
||||||
|
1. Geizhals ist Discovery-Quelle, nicht Decision Engine.
|
||||||
|
2. Die eigenen BuyingProject-Kriterien bleiben die fachliche Quelle der Wahrheit.
|
||||||
|
3. Geizhals wird als möglichst starker Vorfilter genutzt.
|
||||||
|
4. Nicht auf Geizhals abbildbare Kriterien bleiben interne Kriterien.
|
||||||
|
5. Discovery, Ranking und Decision bleiben getrennte fachliche Stufen.
|
||||||
|
6. Externe Filtertaxonomien werden **on demand** gelernt, nicht vollständig vorab katalogisiert.
|
||||||
|
7. URL-Encoding und Filtersyntax dürfen nicht vom LLM erfunden werden.
|
||||||
|
8. Produktidentität wird zunächst konservativ und pragmatisch behandelt.
|
||||||
|
9. Bei Unsicherheit erfolgt keine automatische Zusammenführung.
|
||||||
|
10. Suchläufe speichern strukturierte Zustände, keine HTML-Archive.
|
||||||
|
|
||||||
|
## 3. Scope v0.1
|
||||||
|
|
||||||
|
### Enthalten
|
||||||
|
|
||||||
|
- Geizhals als erste Discovery-Quelle
|
||||||
|
- Pilotkategorie Notebooks
|
||||||
|
- On-Demand-Erfassung der Geizhals-Filtertaxonomie
|
||||||
|
- Speicherung von Source-Kategorien, Filterdefinitionen, Optionen und Wertebereichen
|
||||||
|
- Mapping zwischen Canonical Criteria und Source-Filtern
|
||||||
|
- reproduzierbare Generierung von Geizhals-Such-URLs
|
||||||
|
- periodische SearchRuns
|
||||||
|
- vollständige Ergebnisaufnahme einer aktiven Suche
|
||||||
|
- persistierte SourceProductReferences
|
||||||
|
- Historisierung sichtbarer Attribute und Preise
|
||||||
|
- Diffing zwischen Suchläufen
|
||||||
|
- Status `NEW`, `CHANGED`, `UNCHANGED`, `MISSING`, `DISAPPEARED`
|
||||||
|
- interne Bewertung der Kandidaten
|
||||||
|
- konfigurierbare Shortlist, Default 5
|
||||||
|
- defensives Fehler-/Backoff-Verhalten
|
||||||
|
- Tests
|
||||||
|
|
||||||
|
### Nicht enthalten
|
||||||
|
|
||||||
|
- Frontend-Implementierung
|
||||||
|
- Keepa
|
||||||
|
- Idealo
|
||||||
|
- LLM-Automation
|
||||||
|
- automatische Kaufentscheidung
|
||||||
|
- Kaufautomatisierung
|
||||||
|
- Warenkorbmanagement
|
||||||
|
- Authentifizierung / RBAC
|
||||||
|
- Multi-User-UI
|
||||||
|
- automatische Deep-Enrichment-Quellen
|
||||||
|
- Umgehung von Schutzmechanismen externer Websites
|
||||||
|
|
||||||
|
## 4. Source Taxonomy
|
||||||
|
|
||||||
|
### 4.1 SourceCategory
|
||||||
|
|
||||||
|
Repräsentiert eine Kategorie einer externen Quelle.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"source": "geizhals",
|
||||||
|
"external_category_key": "nb",
|
||||||
|
"name": "Notebooks",
|
||||||
|
"source_url": "https://geizhals.de/?cat=nb",
|
||||||
|
"status": "ACTIVE",
|
||||||
|
"last_verified_at": "2026-10-04T10:00:00Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 SourceFilterDefinition
|
||||||
|
|
||||||
|
Beschreibt einen von Geizhals angebotenen Filter.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"source": "geizhals",
|
||||||
|
"category_key": "nb",
|
||||||
|
"external_filter_key": "17023",
|
||||||
|
"label": "Arbeitsspeicher",
|
||||||
|
"filter_type": "NUMERIC_THRESHOLD",
|
||||||
|
"canonical_unit": "GB",
|
||||||
|
"status": "VERIFIED"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Mögliche `filter_type`-Werte:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BOOLEAN
|
||||||
|
ENUM
|
||||||
|
MULTI_ENUM
|
||||||
|
NUMERIC_RANGE
|
||||||
|
NUMERIC_THRESHOLD
|
||||||
|
TEXT
|
||||||
|
UNKNOWN
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.3 SourceFilterOption
|
||||||
|
|
||||||
|
Für diskrete Filterwerte:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"filter_key": "17023",
|
||||||
|
"source_value": "ab 24GB",
|
||||||
|
"normalized_value": 24,
|
||||||
|
"operator": ">=",
|
||||||
|
"unit": "GB"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.4 SourceFilterRange
|
||||||
|
|
||||||
|
Für Bereiche bzw. frei parametrisierbare Werte:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"filter_key": "price",
|
||||||
|
"minimum": 0,
|
||||||
|
"maximum": null,
|
||||||
|
"step": 1,
|
||||||
|
"unit": "EUR"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. On-Demand Taxonomy Learning
|
||||||
|
|
||||||
|
Die Taxonomie wird nur für tatsächlich verwendete Kategorien gelernt.
|
||||||
|
|
||||||
|
Initial:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Geizhals → Notebooks
|
||||||
|
```
|
||||||
|
|
||||||
|
Später z. B.:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Monitore
|
||||||
|
Fernseher
|
||||||
|
Kameras
|
||||||
|
Rasenmäher
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Ablauf
|
||||||
|
|
||||||
|
```text
|
||||||
|
Kategorie öffnen
|
||||||
|
↓
|
||||||
|
verfügbare Filter erkennen
|
||||||
|
↓
|
||||||
|
Filterdefinitionen speichern
|
||||||
|
↓
|
||||||
|
Optionen / Bereiche erfassen
|
||||||
|
↓
|
||||||
|
URL-Verhalten beobachten
|
||||||
|
↓
|
||||||
|
Mapping erzeugen
|
||||||
|
↓
|
||||||
|
technisch verifizieren
|
||||||
|
```
|
||||||
|
|
||||||
|
### Mapping-Status
|
||||||
|
|
||||||
|
```text
|
||||||
|
DISCOVERED
|
||||||
|
LEARNED_UNVERIFIED
|
||||||
|
VERIFIED
|
||||||
|
BROKEN
|
||||||
|
DISABLED
|
||||||
|
```
|
||||||
|
|
||||||
|
Nur `VERIFIED` darf für automatische produktive Suchläufe verwendet werden.
|
||||||
|
|
||||||
|
### Änderbarkeit
|
||||||
|
|
||||||
|
Die gelernten Mappings sollen später im Frontend administrierbar sein.
|
||||||
|
|
||||||
|
v0.1 benötigt dafür noch keine UI, aber die Persistenz muss Änderungen ermöglichen, ohne Source-Code anzupassen.
|
||||||
|
|
||||||
|
## 6. Canonical Criteria
|
||||||
|
|
||||||
|
Canonical Criteria sind die interne fachliche Darstellung eines Suchkriteriums.
|
||||||
|
|
||||||
|
Sie sollen sich stark an der Source-Taxonomie orientieren, aber nicht fest an Geizhals gekoppelt sein.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"key": "ram_gb",
|
||||||
|
"value_type": "number",
|
||||||
|
"unit": "GB",
|
||||||
|
"semantic": "working_memory"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Ein BuyingProject verwendet daraus konkrete Anforderungen:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"criterion": "ram_gb",
|
||||||
|
"operator": ">=",
|
||||||
|
"value": 24,
|
||||||
|
"mandatory": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Source Mapping
|
||||||
|
|
||||||
|
```text
|
||||||
|
Canonical Criterion
|
||||||
|
ram_gb >= 24
|
||||||
|
|
||||||
|
↓
|
||||||
|
|
||||||
|
Geizhals Mapping
|
||||||
|
filter_key = 17023
|
||||||
|
source_value = "ab 24GB"
|
||||||
|
|
||||||
|
↓
|
||||||
|
|
||||||
|
URL fragment
|
||||||
|
xf=17023_ab+24GB
|
||||||
|
```
|
||||||
|
|
||||||
|
Nicht jedes Canonical Criterion muss ein Source Mapping besitzen.
|
||||||
|
|
||||||
|
Fehlt ein Mapping:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Criterion → INTERNAL_ONLY
|
||||||
|
```
|
||||||
|
|
||||||
|
Das Kriterium wird dann erst nach Discovery intern bewertet.
|
||||||
|
|
||||||
|
## 7. DiscoverySearch
|
||||||
|
|
||||||
|
Eine `DiscoverySearch` gehört zu einem BuyingProject und einer Source.
|
||||||
|
|
||||||
|
Sie enthält mindestens:
|
||||||
|
|
||||||
|
```text
|
||||||
|
id
|
||||||
|
buying_project_id
|
||||||
|
source_id
|
||||||
|
source_category_id
|
||||||
|
criteria_version
|
||||||
|
mapping_version
|
||||||
|
generated_url
|
||||||
|
status
|
||||||
|
schedule_interval_minutes
|
||||||
|
broad_result_threshold
|
||||||
|
created_at
|
||||||
|
updated_at
|
||||||
|
last_successful_run_at
|
||||||
|
```
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
```text
|
||||||
|
schedule_interval_minutes = 180
|
||||||
|
shortlist_size = 5
|
||||||
|
broad_result_threshold = 300
|
||||||
|
```
|
||||||
|
|
||||||
|
## 8. Search URL Generation
|
||||||
|
|
||||||
|
Die URL wird ausschließlich aus:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BuyingProject Criteria
|
||||||
|
+
|
||||||
|
VERIFIED CriterionMappings
|
||||||
|
+
|
||||||
|
SourceCategory
|
||||||
|
+
|
||||||
|
konfigurierbaren Source-Optionen
|
||||||
|
```
|
||||||
|
|
||||||
|
generiert.
|
||||||
|
|
||||||
|
Die URL ist abgeleitet, nicht die fachliche Quelle der Wahrheit.
|
||||||
|
|
||||||
|
Eine Änderung der BuyingProject-Kriterien erzeugt eine neue `criteria_version`.
|
||||||
|
|
||||||
|
Wenn die gespeicherte DiscoverySearch nicht mehr zur aktuellen Criteria-/Mapping-Version passt:
|
||||||
|
|
||||||
|
```text
|
||||||
|
status = OUT_OF_SYNC
|
||||||
|
```
|
||||||
|
|
||||||
|
Danach muss die URL neu generiert werden.
|
||||||
|
|
||||||
|
## 9. Ergebnisumfang
|
||||||
|
|
||||||
|
Aktive Suchläufe werden grundsätzlich vollständig eingelesen.
|
||||||
|
|
||||||
|
Keine stille Begrenzung auf Top-N.
|
||||||
|
|
||||||
|
Nur vollständige Ergebnismengen erlauben belastbare Erkennung von:
|
||||||
|
|
||||||
|
```text
|
||||||
|
NEW
|
||||||
|
CHANGED
|
||||||
|
MISSING
|
||||||
|
DISAPPEARED
|
||||||
|
```
|
||||||
|
|
||||||
|
### TOO_BROAD
|
||||||
|
|
||||||
|
Überschreitet die Treffermenge den konfigurierten Schwellwert:
|
||||||
|
|
||||||
|
```text
|
||||||
|
result_count > broad_result_threshold
|
||||||
|
```
|
||||||
|
|
||||||
|
wird die Suche markiert:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TOO_BROAD
|
||||||
|
```
|
||||||
|
|
||||||
|
Die Suche darf technisch weiter ausgeführt werden, erzeugt aber einen Hinweis, dass die Source-Kriterien geschärft werden sollten.
|
||||||
|
|
||||||
|
Spätere Versionen dürfen breite Suchen differenziert behandeln, etwa seltenere Full-Reconciliation.
|
||||||
|
|
||||||
|
## 10. SearchRun
|
||||||
|
|
||||||
|
Jeder Abruf erzeugt einen unveränderlichen `SearchRun`.
|
||||||
|
|
||||||
|
Felder:
|
||||||
|
|
||||||
|
```text
|
||||||
|
id
|
||||||
|
discovery_search_id
|
||||||
|
criteria_version
|
||||||
|
mapping_version
|
||||||
|
generated_url
|
||||||
|
started_at
|
||||||
|
finished_at
|
||||||
|
status
|
||||||
|
result_count
|
||||||
|
page_count
|
||||||
|
http_status_summary
|
||||||
|
error_code
|
||||||
|
```
|
||||||
|
|
||||||
|
Status:
|
||||||
|
|
||||||
|
```text
|
||||||
|
RUNNING
|
||||||
|
SUCCESS
|
||||||
|
PARTIAL
|
||||||
|
FAILED
|
||||||
|
BLOCKED
|
||||||
|
```
|
||||||
|
|
||||||
|
Es wird kein HTML-Snapshot gespeichert.
|
||||||
|
|
||||||
|
## 11. SourceProductReference
|
||||||
|
|
||||||
|
Ein Treffer einer externen Quelle wird als `SourceProductReference` dauerhaft angelegt.
|
||||||
|
|
||||||
|
Mindestens verfügbare Felder:
|
||||||
|
|
||||||
|
```text
|
||||||
|
id
|
||||||
|
source_id
|
||||||
|
source_category_id
|
||||||
|
source_url
|
||||||
|
source_title
|
||||||
|
manufacturer
|
||||||
|
model
|
||||||
|
manufacturer_sku
|
||||||
|
gtin_ean
|
||||||
|
normalized_spec_fingerprint
|
||||||
|
first_seen_at
|
||||||
|
last_seen_at
|
||||||
|
status
|
||||||
|
```
|
||||||
|
|
||||||
|
Keines dieser Felder außer der internen UUID wird pauschal als garantiert stabil betrachtet.
|
||||||
|
|
||||||
|
## 12. Produktidentität – Pilotansatz
|
||||||
|
|
||||||
|
v0.1 implementiert bewusst keine komplexe globale Identity Engine.
|
||||||
|
|
||||||
|
Für den Notebook-Pilot werden folgende Signale verwendet:
|
||||||
|
|
||||||
|
1. vorhandene stabile Source-ID, falls entdeckt
|
||||||
|
2. GTIN/EAN, falls vorhanden
|
||||||
|
3. Hersteller-SKU, falls vorhanden
|
||||||
|
4. kanonische Source-URL
|
||||||
|
5. normalisierter Titel
|
||||||
|
6. Hersteller + Modell
|
||||||
|
7. differenzierende technische Specs
|
||||||
|
|
||||||
|
Für Notebooks kann der initiale Fingerprint beispielsweise enthalten:
|
||||||
|
|
||||||
|
```text
|
||||||
|
manufacturer
|
||||||
|
model
|
||||||
|
cpu
|
||||||
|
ram
|
||||||
|
storage
|
||||||
|
display_size
|
||||||
|
gpu
|
||||||
|
```
|
||||||
|
|
||||||
|
### Matching-Regeln
|
||||||
|
|
||||||
|
- Hohe Sicherheit → existierende SourceProductReference verwenden
|
||||||
|
- Unsicherheit → `MATCH_UNCERTAIN`
|
||||||
|
- Keine sichere Übereinstimmung → neue SourceProductReference
|
||||||
|
- niemals zwei Produkte allein aufgrund ähnlicher Namen automatisch verschmelzen
|
||||||
|
|
||||||
|
Die tatsächliche Stabilität der Identitätsmerkmale wird im Notebook-Pilot beobachtet und danach verfeinert.
|
||||||
|
|
||||||
|
## 13. Produktbeobachtungen und Historie
|
||||||
|
|
||||||
|
Technische Eigenschaften und Preise werden nicht einfach überschrieben.
|
||||||
|
|
||||||
|
Jeder SearchRun kann neue Beobachtungen erzeugen.
|
||||||
|
|
||||||
|
### ProductObservation
|
||||||
|
|
||||||
|
```text
|
||||||
|
source_product_reference_id
|
||||||
|
search_run_id
|
||||||
|
observed_at
|
||||||
|
raw_title
|
||||||
|
best_price
|
||||||
|
currency
|
||||||
|
raw_attributes
|
||||||
|
normalized_attributes
|
||||||
|
availability
|
||||||
|
seller_count
|
||||||
|
```
|
||||||
|
|
||||||
|
Veränderte Werte bleiben historisch nachvollziehbar.
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Preis: 1499 → 1299
|
||||||
|
RAM: 16 → 32
|
||||||
|
Anbieterzahl: 8 → 12
|
||||||
|
Verfügbarkeit geändert
|
||||||
|
```
|
||||||
|
|
||||||
|
Bei technischen Attributänderungen muss das System berücksichtigen, dass auch fehlerhaftes Matching vorliegen könnte.
|
||||||
|
|
||||||
|
## 14. Diffing
|
||||||
|
|
||||||
|
Jeder erfolgreiche Full SearchRun wird mit dem vorherigen erfolgreichen Full SearchRun derselben DiscoverySearch verglichen.
|
||||||
|
|
||||||
|
### Status
|
||||||
|
|
||||||
|
```text
|
||||||
|
NEW
|
||||||
|
CHANGED
|
||||||
|
UNCHANGED
|
||||||
|
MISSING
|
||||||
|
DISAPPEARED
|
||||||
|
```
|
||||||
|
|
||||||
|
### Regeln
|
||||||
|
|
||||||
|
**NEW**
|
||||||
|
Produkt war im vorherigen vollständigen Lauf nicht vorhanden.
|
||||||
|
|
||||||
|
**CHANGED**
|
||||||
|
Produkt vorhanden, aber mindestens ein relevantes beobachtetes Feld hat sich verändert.
|
||||||
|
|
||||||
|
**UNCHANGED**
|
||||||
|
Produkt vorhanden und relevante Felder unverändert.
|
||||||
|
|
||||||
|
**MISSING**
|
||||||
|
Produkt fehlt in einem vollständigen Lauf.
|
||||||
|
|
||||||
|
**DISAPPEARED**
|
||||||
|
Produkt fehlt in drei aufeinanderfolgenden vollständigen Läufen.
|
||||||
|
|
||||||
|
Produkte werden nicht gelöscht.
|
||||||
|
|
||||||
|
Historie bleibt erhalten.
|
||||||
|
|
||||||
|
Ein später erneut auftauchendes Produkt wird reaktiviert und nicht als neues Produkt dupliziert, sofern das Matching ausreichend sicher ist.
|
||||||
|
|
||||||
|
## 15. Lifecycle eines BuyingProjects
|
||||||
|
|
||||||
|
Während ein BuyingProject aktiv ist:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ACTIVE
|
||||||
|
↓
|
||||||
|
DiscoverySearches laufen
|
||||||
|
↓
|
||||||
|
Produkte + Historie wachsen
|
||||||
|
↓
|
||||||
|
Ranking + Shortlist
|
||||||
|
```
|
||||||
|
|
||||||
|
Nach finalem Kauf:
|
||||||
|
|
||||||
|
```text
|
||||||
|
PURCHASED
|
||||||
|
↓
|
||||||
|
BuyingProject archivieren
|
||||||
|
↓
|
||||||
|
Discovery stoppen
|
||||||
|
```
|
||||||
|
|
||||||
|
Produkte, Beobachtungen und Evaluationen können mit dem Projekt archiviert bzw. aus aktiven Ansichten entfernt werden.
|
||||||
|
|
||||||
|
Keine physische Löschung historischer Daten als Standardverhalten.
|
||||||
|
|
||||||
|
## 16. Ranking
|
||||||
|
|
||||||
|
Nach Discovery übernimmt ausschließlich die interne Buying Engine.
|
||||||
|
|
||||||
|
```text
|
||||||
|
Source Results
|
||||||
|
↓
|
||||||
|
interne Hard Constraints
|
||||||
|
↓
|
||||||
|
ProductScore
|
||||||
|
↓
|
||||||
|
Ranking
|
||||||
|
```
|
||||||
|
|
||||||
|
Geizhals-Sortierung darf für die technische Ausgabe genutzt werden, ist aber keine fachliche Ranking-Entscheidung.
|
||||||
|
|
||||||
|
## 17. Shortlist
|
||||||
|
|
||||||
|
Nach internem Ranking wird eine Shortlist gebildet.
|
||||||
|
|
||||||
|
Default:
|
||||||
|
|
||||||
|
```text
|
||||||
|
shortlist_size = 5
|
||||||
|
```
|
||||||
|
|
||||||
|
Der Wert ist pro BuyingProject konfigurierbar.
|
||||||
|
|
||||||
|
Shortlist-Kandidaten erhalten:
|
||||||
|
|
||||||
|
```text
|
||||||
|
SHORTLISTED
|
||||||
|
```
|
||||||
|
|
||||||
|
Andere gültige Kandidaten:
|
||||||
|
|
||||||
|
```text
|
||||||
|
CANDIDATE
|
||||||
|
```
|
||||||
|
|
||||||
|
Die spätere Deep-Enrichment-Stufe arbeitet primär auf der Shortlist.
|
||||||
|
|
||||||
|
## 18. Deep Enrichment
|
||||||
|
|
||||||
|
Nicht Bestandteil dieses Implementierungsslices.
|
||||||
|
|
||||||
|
Zielbild:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Shortlist
|
||||||
|
↓
|
||||||
|
Herstellerdaten
|
||||||
|
Tests
|
||||||
|
Benchmarks
|
||||||
|
Reviews
|
||||||
|
weitere Händler
|
||||||
|
Preisportale
|
||||||
|
Qualitätsinformationen
|
||||||
|
↓
|
||||||
|
erneute Bewertung
|
||||||
|
```
|
||||||
|
|
||||||
|
Discovery darf daher nicht voraussetzen, dass Geizhals alle für die finale Kaufentscheidung notwendigen Daten liefert.
|
||||||
|
|
||||||
|
## 19. Discovery, Ranking und Decision
|
||||||
|
|
||||||
|
Die drei fachlichen Stufen müssen getrennt bleiben.
|
||||||
|
|
||||||
|
### Discovery
|
||||||
|
|
||||||
|
```text
|
||||||
|
Welche Produkte kommen potenziell infrage?
|
||||||
|
```
|
||||||
|
|
||||||
|
### Ranking
|
||||||
|
|
||||||
|
```text
|
||||||
|
Welche dieser Produkte passen am besten zum BuyingProject?
|
||||||
|
```
|
||||||
|
|
||||||
|
### Decision
|
||||||
|
|
||||||
|
```text
|
||||||
|
Ist jetzt der richtige Zeitpunkt, Kandidat X zu kaufen?
|
||||||
|
```
|
||||||
|
|
||||||
|
Decision wird später u. a. berücksichtigen können:
|
||||||
|
|
||||||
|
- Zielpreis
|
||||||
|
- Deadline
|
||||||
|
- historische Preisentwicklung
|
||||||
|
- aktuelle Verfügbarkeit
|
||||||
|
- Product Fit
|
||||||
|
- Offer Fit
|
||||||
|
- Alternativen
|
||||||
|
- Beschaffungsdringlichkeit
|
||||||
|
|
||||||
|
Decision ist nicht Bestandteil dieses Slices.
|
||||||
|
|
||||||
|
## 20. Scheduling und defensiver Abruf
|
||||||
|
|
||||||
|
Default:
|
||||||
|
|
||||||
|
```text
|
||||||
|
alle 180 Minuten
|
||||||
|
```
|
||||||
|
|
||||||
|
pro DiscoverySearch konfigurierbar.
|
||||||
|
|
||||||
|
### Anforderungen
|
||||||
|
|
||||||
|
- keine parallelen Massenabrufe
|
||||||
|
- Pagination sequenziell oder stark begrenzt parallel
|
||||||
|
- Caching, wo sinnvoll
|
||||||
|
- Timeout
|
||||||
|
- Retry mit exponentiellem Backoff
|
||||||
|
- Jitter
|
||||||
|
- Stop bei wiederholten Fehlern
|
||||||
|
- keine Umgehung von Captchas oder Bot-Schutz
|
||||||
|
|
||||||
|
### Reaktion
|
||||||
|
|
||||||
|
**HTTP 429**
|
||||||
|
|
||||||
|
```text
|
||||||
|
BACKOFF
|
||||||
|
```
|
||||||
|
|
||||||
|
**HTTP 403**
|
||||||
|
|
||||||
|
```text
|
||||||
|
BLOCKED
|
||||||
|
```
|
||||||
|
|
||||||
|
**Captcha / Challenge erkannt**
|
||||||
|
|
||||||
|
```text
|
||||||
|
BLOCKED
|
||||||
|
```
|
||||||
|
|
||||||
|
Automatische weitere Abrufe der betroffenen DiscoverySearch werden pausiert.
|
||||||
|
|
||||||
|
Keine Schutzmechanismen umgehen.
|
||||||
|
|
||||||
|
## 21. Source Adapter Contract
|
||||||
|
|
||||||
|
Ein Discovery-Adapter soll mindestens folgende Fähigkeiten kapseln:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class DiscoverySourceAdapter(Protocol):
|
||||||
|
def discover_taxonomy(category_ref): ...
|
||||||
|
def build_query(criteria, mappings): ...
|
||||||
|
def fetch_results(query, pagination): ...
|
||||||
|
def parse_results(response): ...
|
||||||
|
def normalize_result(result): ...
|
||||||
|
```
|
||||||
|
|
||||||
|
Taxonomy Learning und Result Retrieval dürfen intern getrennte Services sein.
|
||||||
|
|
||||||
|
Der Rest des Systems darf keine Geizhals-spezifische URL- oder HTML-Logik kennen.
|
||||||
|
|
||||||
|
## 22. Services
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TaxonomyService
|
||||||
|
CriterionMappingService
|
||||||
|
DiscoverySearchService
|
||||||
|
SearchExecutionService
|
||||||
|
SourceProductIdentityService
|
||||||
|
SearchDiffService
|
||||||
|
CandidateRankingService
|
||||||
|
ShortlistService
|
||||||
|
```
|
||||||
|
|
||||||
|
Keine Businesslogik in n8n.
|
||||||
|
|
||||||
|
n8n kann später den Scheduler bzw. externe Events orchestrieren.
|
||||||
|
|
||||||
|
## 23. Persistenz
|
||||||
|
|
||||||
|
Neue bzw. ergänzende Tabellen können sein:
|
||||||
|
|
||||||
|
```text
|
||||||
|
source_categories
|
||||||
|
source_filter_definitions
|
||||||
|
source_filter_options
|
||||||
|
source_filter_ranges
|
||||||
|
criterion_definitions
|
||||||
|
criterion_mappings
|
||||||
|
discovery_searches
|
||||||
|
search_runs
|
||||||
|
source_product_references
|
||||||
|
product_observations
|
||||||
|
project_candidates
|
||||||
|
candidate_status_history
|
||||||
|
```
|
||||||
|
|
||||||
|
Bestehende globale Entitäten aus der Hauptspezifikation bleiben bestehen:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Product
|
||||||
|
ProductVariant
|
||||||
|
Offer
|
||||||
|
PriceObservation
|
||||||
|
```
|
||||||
|
|
||||||
|
Diese dürfen nicht unnötig dupliziert werden.
|
||||||
|
|
||||||
|
Der Discovery-Slice kann zunächst mit SourceProductReference arbeiten und erst bei ausreichender Identität eine Verknüpfung zu canonical Product/ProductVariant herstellen.
|
||||||
|
|
||||||
|
## 24. API – minimaler Vertrag
|
||||||
|
|
||||||
|
### Taxonomie
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /api/v1/sources/geizhals/categories/{category}/discover-taxonomy
|
||||||
|
GET /api/v1/sources/geizhals/categories/{category}/filters
|
||||||
|
```
|
||||||
|
|
||||||
|
### Mapping
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/criterion-mappings
|
||||||
|
POST /api/v1/criterion-mappings
|
||||||
|
PATCH /api/v1/criterion-mappings/{id}
|
||||||
|
```
|
||||||
|
|
||||||
|
### DiscoverySearch
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /api/v1/discovery-searches
|
||||||
|
GET /api/v1/discovery-searches/{id}
|
||||||
|
POST /api/v1/discovery-searches/{id}/generate-query
|
||||||
|
POST /api/v1/discovery-searches/{id}/run
|
||||||
|
```
|
||||||
|
|
||||||
|
### SearchRuns
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/discovery-searches/{id}/runs
|
||||||
|
GET /api/v1/search-runs/{id}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Candidates
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/buying-projects/{id}/candidates
|
||||||
|
GET /api/v1/buying-projects/{id}/shortlist
|
||||||
|
```
|
||||||
|
|
||||||
|
Falls BuyingProject im aktuellen Code noch nicht existiert, muss die Integration mit der im Architecture Addendum vorgesehenen nächsten Domain-Erweiterung abgestimmt werden.
|
||||||
|
|
||||||
|
## 25. Tests
|
||||||
|
|
||||||
|
### Taxonomy
|
||||||
|
|
||||||
|
- Notebook-Kategorie kann on demand eingelesen werden
|
||||||
|
- Filterdefinitionen werden reproduzierbar gespeichert
|
||||||
|
- Optionen werden nicht dupliziert
|
||||||
|
- Filteränderungen werden erkannt
|
||||||
|
- BROKEN-Mapping wird nicht produktiv verwendet
|
||||||
|
|
||||||
|
### Query Generation
|
||||||
|
|
||||||
|
- identische Criteria + Mapping-Version erzeugen identische URL
|
||||||
|
- RAM >= 24 erzeugt erwartetes Geizhals-Fragment
|
||||||
|
- Preisbereich korrekt
|
||||||
|
- mehrere Filter kombinierbar
|
||||||
|
- INTERNAL_ONLY-Kriterien verändern die Source-URL nicht
|
||||||
|
- veraltete Mapping-Version markiert Search als OUT_OF_SYNC
|
||||||
|
|
||||||
|
### Search Execution
|
||||||
|
|
||||||
|
- Pagination vollständig
|
||||||
|
- Ergebniszahl korrekt
|
||||||
|
- TOO_BROAD bei Überschreitung
|
||||||
|
- kein stilles Top-N-Abschneiden
|
||||||
|
- 429 → Backoff
|
||||||
|
- 403/Captcha → BLOCKED
|
||||||
|
|
||||||
|
### Identity
|
||||||
|
|
||||||
|
- identische SKU wird wiedererkannt
|
||||||
|
- identische EAN wird wiedererkannt
|
||||||
|
- gleiche URL wird wiedererkannt
|
||||||
|
- leicht geänderter Titel bei stabilen Specs wird erkannt
|
||||||
|
- unsicheres Match erzeugt MATCH_UNCERTAIN
|
||||||
|
- ähnliche Produktnamen werden nicht automatisch verschmolzen
|
||||||
|
|
||||||
|
### Diffing
|
||||||
|
|
||||||
|
- neuer Treffer → NEW
|
||||||
|
- geänderter Preis → CHANGED
|
||||||
|
- unverändert → UNCHANGED
|
||||||
|
- einmal fehlend → MISSING
|
||||||
|
- dreimal fehlend → DISAPPEARED
|
||||||
|
- Wiederauftauchen reaktiviert bestehende Referenz
|
||||||
|
|
||||||
|
### Historie
|
||||||
|
|
||||||
|
- Preisänderungen bleiben historisiert
|
||||||
|
- technische Änderungen bleiben historisiert
|
||||||
|
- alte SearchRuns bleiben unverändert
|
||||||
|
- Projektarchivierung löscht keine History
|
||||||
|
|
||||||
|
### Ranking
|
||||||
|
|
||||||
|
- nur interne Engine bestimmt Ranking
|
||||||
|
- Source-Sortierung beeinflusst ProductScore nicht
|
||||||
|
- Shortlistgröße Default 5
|
||||||
|
- Shortlistgröße konfigurierbar
|
||||||
|
|
||||||
|
## 26. Implementierungsreihenfolge
|
||||||
|
|
||||||
|
1. Bestehende Implementierung gegen diese Spec und das Architecture Addendum prüfen.
|
||||||
|
2. Notwendige BuyingProject-/ProjectCandidate-Minimalstruktur ergänzen, falls für Discovery erforderlich.
|
||||||
|
3. Source Taxonomy + Mapping persistieren.
|
||||||
|
4. Geizhals-Taxonomie-Pilot für Notebooks implementieren.
|
||||||
|
5. reproduzierbare Query Generation implementieren.
|
||||||
|
6. SearchRun + Pagination implementieren.
|
||||||
|
7. SourceProductReference + ProductObservation implementieren.
|
||||||
|
8. Identity Pilot implementieren.
|
||||||
|
9. Diffing implementieren.
|
||||||
|
10. interne Ranking-Engine anbinden.
|
||||||
|
11. Shortlist bilden.
|
||||||
|
12. Scheduling-Schnittstelle vorbereiten bzw. internen Scheduler minimal anbinden.
|
||||||
|
13. Tests vollständig ausführen.
|
||||||
|
14. Notebook-Pilot mit realer Suche validieren.
|
||||||
|
|
||||||
|
## 27. Definition of Done
|
||||||
|
|
||||||
|
- [ ] Geizhals-Notebook-Taxonomie wird on demand eingelesen und gespeichert.
|
||||||
|
- [ ] Source-Filter und Optionen werden persistiert.
|
||||||
|
- [ ] Filter-Mappings besitzen Lifecycle-Status.
|
||||||
|
- [ ] BuyingProject-Kriterien erzeugen reproduzierbar eine Geizhals-URL.
|
||||||
|
- [ ] Nicht abbildbare Kriterien bleiben INTERNAL_ONLY.
|
||||||
|
- [ ] Aktive Suche kann vollständig ausgeführt werden.
|
||||||
|
- [ ] Pagination wird vollständig verarbeitet.
|
||||||
|
- [ ] TOO_BROAD wird erkannt.
|
||||||
|
- [ ] SearchRuns werden gespeichert.
|
||||||
|
- [ ] Produkte werden dauerhaft als SourceProductReference angelegt.
|
||||||
|
- [ ] Preise und relevante Attribute werden historisiert.
|
||||||
|
- [ ] NEW / CHANGED / UNCHANGED / MISSING / DISAPPEARED funktionieren.
|
||||||
|
- [ ] DISAPPEARED erst nach drei aufeinanderfolgenden fehlenden Full Runs.
|
||||||
|
- [ ] Unsichere Produktidentität wird nicht automatisch verschmolzen.
|
||||||
|
- [ ] interne Engine bewertet Kandidaten.
|
||||||
|
- [ ] Shortlist Default 5 funktioniert.
|
||||||
|
- [ ] Suchfrequenz ist konfigurierbar, Default 180 Minuten.
|
||||||
|
- [ ] 403/429/Captcha führen zu defensivem Stop/Backoff.
|
||||||
|
- [ ] keine HTML-Archive.
|
||||||
|
- [ ] keine Notebook-spezifische Businesslogik außerhalb des Geizhals-Pilotmappings.
|
||||||
|
- [ ] Tests grün.
|
||||||
|
- [ ] Ergebnisse des Notebook-Piloten dokumentiert.
|
||||||
|
|
||||||
|
# Coding-Agent-Arbeitsauftrag
|
||||||
|
|
||||||
|
Verwende als verbindliche Anforderungen:
|
||||||
|
|
||||||
|
1. `buying-agent-v0.1-spec-v1.1.md`
|
||||||
|
2. `buying-agent-architecture-addendum-v1.md`
|
||||||
|
3. `buying-agent-source-discovery-spec-v0.1.md`
|
||||||
|
|
||||||
|
Die neue Source-/Discovery-Spec ergänzt die bestehenden Dokumente. Bei Konflikten in ihrem Fachbereich hat sie Vorrang.
|
||||||
|
|
||||||
|
## Vorgehen
|
||||||
|
|
||||||
|
### 1. Zuerst analysieren
|
||||||
|
|
||||||
|
Prüfe den aktuellen Repository-Stand gegen alle drei Dokumente.
|
||||||
|
|
||||||
|
Berichte vor größeren Änderungen:
|
||||||
|
|
||||||
|
- welche Teile bereits kompatibel sind,
|
||||||
|
- welche Domain-/DB-/API-Erweiterungen erforderlich sind,
|
||||||
|
- ob bestehende Implementierung refaktoriert werden muss,
|
||||||
|
- welche Änderungen rein additiv möglich sind.
|
||||||
|
|
||||||
|
### 2. Minimal integrieren
|
||||||
|
|
||||||
|
Implementiere nur den beschriebenen Source-/Discovery-Slice.
|
||||||
|
|
||||||
|
Nicht implementieren:
|
||||||
|
|
||||||
|
- Frontend
|
||||||
|
- Keepa
|
||||||
|
- Idealo
|
||||||
|
- LLM-Automation
|
||||||
|
- Auth/RBAC
|
||||||
|
- Kaufautomatisierung
|
||||||
|
- Warenkorb
|
||||||
|
|
||||||
|
### 3. Besonders beachten
|
||||||
|
|
||||||
|
- nicht notebook-spezifisch modellieren
|
||||||
|
- Geizhals-spezifische Logik ausschließlich im Source Adapter / Mapping Layer
|
||||||
|
- URL-Encoding nur aus verifizierten Source-Mappings
|
||||||
|
- vollständige SearchRuns, kein stilles Top-N-Limit
|
||||||
|
- konservatives Product Matching
|
||||||
|
- keine automatische Zusammenführung bei Unsicherheit
|
||||||
|
- alle Änderungen historisieren
|
||||||
|
- Discovery, Ranking und Decision getrennt halten
|
||||||
|
|
||||||
|
### 4. Tests
|
||||||
|
|
||||||
|
Führe Unit-, Integration- und End-to-End-Tests aus.
|
||||||
|
|
||||||
|
Mindestens:
|
||||||
|
|
||||||
|
- Taxonomy Discovery
|
||||||
|
- Query Generation
|
||||||
|
- Pagination
|
||||||
|
- Identity Matching
|
||||||
|
- Diffing
|
||||||
|
- History
|
||||||
|
- Ranking
|
||||||
|
- Shortlist
|
||||||
|
- Backoff / BLOCKED
|
||||||
|
|
||||||
|
### 5. Abschlussbericht
|
||||||
|
|
||||||
|
Dokumentiere:
|
||||||
|
|
||||||
|
- geänderte Dateien
|
||||||
|
- neue Migrationen
|
||||||
|
- neue API-Endpunkte
|
||||||
|
- Testergebnisse
|
||||||
|
- real durchgeführten Notebook-Pilot
|
||||||
|
- erkannte Geizhals-Eigenheiten
|
||||||
|
- verbleibende Risiken
|
||||||
|
- insbesondere Beobachtungen zur Stabilität der Produktidentität
|
||||||
61
README.md
61
README.md
|
|
@ -1,28 +1,48 @@
|
||||||
# BuyingAgent
|
# BuyingAgent
|
||||||
|
|
||||||
Backend für einen generischen Buying-/Deal-Agenten. Der erste fachliche Schnitt ist in `.specs/buying-agent-v0.1-spec.md` beschrieben.
|
Backend für einen generischen Buying-/Deal-Agenten. Das System bewertet, wie gut eine Produktvariante zu einem Profil passt und wie attraktiv ein konkretes Angebot preislich ist. Es kauft nicht.
|
||||||
|
|
||||||
Entwicklung auf `develop`. Push nach `develop` baut die Dev-Umgebung auf dem Raspberry Pi. Ein Merge-Commit nach `main` deployt Production. Details: `docs/DEPLOYMENT.md`.
|
v0.1 ist ein Walking Skeleton: Profil → Produkt → Variante → Angebot → Preisbeobachtungen → Evaluation. Notebooks sind der erste Anwendungsfall, nicht das Datenmodell.
|
||||||
|
|
||||||
|
**Vorgaben:** `docs/buying-agent-v0.1-spec-v1.1.md` (fachlich) und `docs/buying-agent-architecture-addendum-v1.md` (Architektur, hat Vorrang bei Strukturfragen).
|
||||||
|
|
||||||
|
## Monorepo
|
||||||
|
|
||||||
|
```text
|
||||||
|
backend/ System of Record: Domain, Scoring, Persistenz, /api/v1
|
||||||
|
frontend/ Platzhalter — keine UI in v0.1
|
||||||
|
docs/ Spezifikationen und Deployment
|
||||||
|
docker-compose.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Entwicklung auf `develop`. Push nach `develop` baut Dev auf dem Raspberry Pi. Merge `--no-ff` nach `main` deployt Production. Details: `docs/DEPLOYMENT.md`.
|
||||||
|
|
||||||
## Stack
|
## Stack
|
||||||
|
|
||||||
Python 3.13, FastAPI, Pydantic 2, SQLAlchemy 2, Alembic, PostgreSQL 17, pytest, Docker Compose. Build-Tool: [uv](https://docs.astral.sh/uv/) 0.10.11.
|
Python 3.13, FastAPI, Pydantic 2, SQLAlchemy 2 (sync, psycopg 3), Alembic, PostgreSQL 17, pytest, Docker Compose. Build-Tool: uv 0.10.11.
|
||||||
|
|
||||||
|
Abhängigkeiten: HTTP → Services → Domain/Scoring → Repositories → PostgreSQL. `domain` und `scoring` importieren weder FastAPI noch SQLAlchemy.
|
||||||
|
|
||||||
## Lokal starten
|
## Lokal starten
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
Copy-Item .env.example .env
|
Copy-Item .env.example .env
|
||||||
docker compose up --build -d
|
docker compose up --build -d
|
||||||
|
docker compose exec api python -m scripts.demo
|
||||||
```
|
```
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
docker compose up --build -d
|
docker compose up --build -d
|
||||||
|
docker compose exec api python -m scripts.demo
|
||||||
```
|
```
|
||||||
|
|
||||||
API: `http://127.0.0.1:8000`
|
API: `http://127.0.0.1:8000`
|
||||||
|
OpenAPI: `http://127.0.0.1:8000/docs`
|
||||||
Live: `GET /health/live`
|
Live: `GET /health/live`
|
||||||
Ready: `GET /health/ready` (PostgreSQL erreichbar und Alembic auf Head)
|
Ready: `GET /health/ready`
|
||||||
|
|
||||||
|
Die Demo legt Definitionen, Profil und die Fälle A–D über HTTP an und wertet sie mit `as_of=2026-10-03T12:00:00Z` aus. Wiederholter Lauf erzeugt keine Duplikate.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
|
|
@ -32,7 +52,38 @@ Lokal, isolierter Stack (tmpfs, kein Dev-/Prod-Volume):
|
||||||
docker compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from api
|
docker compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from api
|
||||||
```
|
```
|
||||||
|
|
||||||
Auf dem Server nach Dev-Deploy (Gitea `test.yml`, analog Kanshō): pytest gegen `buyingagent_test` auf dem Dev-Postgres plus read-only Smoke gegen `http://192.168.2.49:8095/health/ready`.
|
Nur Unit-Tests (ohne PostgreSQL), im Verzeichnis `backend/`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
pytest tests/unit -ra --tb=short
|
||||||
|
```
|
||||||
|
|
||||||
|
Auf dem Server nach Dev-Deploy (Gitea `test.yml`): pytest gegen `buyingagent_test` auf dem Dev-Postgres plus Smoke gegen `http://192.168.2.49:8095/health/ready`.
|
||||||
|
|
||||||
|
## Datenmodell (global)
|
||||||
|
|
||||||
|
Product, ProductVariant, Offer, PriceObservation, AttributeDefinition und Source sind **systemweit** und haben keine `user_id`/`workspace_id`. Workspace, BuyingProject und Benutzer kommen später und dürfen diese Daten nicht duplizieren.
|
||||||
|
|
||||||
|
Profile und Evaluationen können bereits parallel existieren; es gibt kein globales „aktuelles Kaufvorhaben“.
|
||||||
|
|
||||||
|
## Scoring `generic-v1`
|
||||||
|
|
||||||
|
- Preference-Strategie nur `numeric_linear`. Unbekannte Strategien werden beim Profil abgewiesen.
|
||||||
|
- `product_eligibility`, `offer_eligibility` und `overall_eligibility` sind getrennt.
|
||||||
|
- PriceScore ist eine reine Funktion über `current_price`, `reference_history` und `budget`. v0.1 setzt `price_reference_scope=offer`.
|
||||||
|
- DealScore nur bei `overall_eligibility=ELIGIBLE` und vorhandenem PriceScore.
|
||||||
|
- Klassifikation am **ungerundeten** DealScore; Anzeige auf zwei Dezimalen, `ROUND_HALF_UP`.
|
||||||
|
- Geld intern `Decimal`/`NUMERIC(12,2)`, in der API als Dezimalstring.
|
||||||
|
|
||||||
|
Referenzfall A (synthetisch): ProductScore 90.21, PriceScore 98.24, DealScore 93.42, `EXCEPTIONAL`. Das ist keine Marktaussage.
|
||||||
|
|
||||||
|
## Fehler
|
||||||
|
|
||||||
|
Einheitliches Format `{error: {code, message, details}}`. 404 fehlende ID, 409 Konflikt/abweichender Wiederholungsimport, 422 ungültige Payload. Eine fachlich negative Evaluation ist HTTP 201.
|
||||||
|
|
||||||
|
## Grenzen von v0.1
|
||||||
|
|
||||||
|
Keine Web-UI, keine Authentifizierung, keine Workspaces, kein BuyingProject, kein n8n, keine externen Quellen, keine Auto-Discovery, keine Wechselkurse, keine Variant-/Market-Level-Preisaggregation, keine Plugin-Infrastruktur.
|
||||||
|
|
||||||
## Server
|
## Server
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
__version__ = "0.1.0"
|
|
||||||
10
app/main.py
10
app/main.py
|
|
@ -1,10 +0,0 @@
|
||||||
from fastapi import FastAPI
|
|
||||||
|
|
||||||
from app.api.health import router as health_router
|
|
||||||
|
|
||||||
app = FastAPI(
|
|
||||||
title="Buying Agent",
|
|
||||||
version="0.1.0",
|
|
||||||
description="Walking skeleton for the buying and deal evaluation API.",
|
|
||||||
)
|
|
||||||
app.include_router(health_router)
|
|
||||||
10
backend/.dockerignore
Normal file
10
backend/.dockerignore
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
.git
|
||||||
|
.venv
|
||||||
|
.pytest_cache
|
||||||
|
.mypy_cache
|
||||||
|
.ruff_cache
|
||||||
|
__pycache__
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.egg-info
|
||||||
|
.tests-tmp
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
[alembic]
|
[alembic]
|
||||||
script_location = migrations
|
script_location = migrations
|
||||||
prepend_sys_path = .
|
prepend_sys_path = .
|
||||||
|
path_separator = os
|
||||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||||
|
|
||||||
[loggers]
|
[loggers]
|
||||||
0
backend/app/__init__.py
Normal file
0
backend/app/__init__.py
Normal file
0
backend/app/api/__init__.py
Normal file
0
backend/app/api/__init__.py
Normal file
125
backend/app/api/discovery.py
Normal file
125
backend/app/api/discovery.py
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.schemas import (
|
||||||
|
CriterionDefinitionIn,
|
||||||
|
CriterionMappingIn,
|
||||||
|
CriterionMappingPatch,
|
||||||
|
DiscoverySearchIn,
|
||||||
|
DiscoverySearchPatch,
|
||||||
|
SourceAttributeMappingIn,
|
||||||
|
SourceAttributeMappingPatch,
|
||||||
|
)
|
||||||
|
from app.persistence.db import get_session
|
||||||
|
from app.services.catalog_service import CatalogService
|
||||||
|
from app.services.discovery_service import (
|
||||||
|
CriterionDefinitionService,
|
||||||
|
CriterionMappingService,
|
||||||
|
DiscoverySearchService,
|
||||||
|
SourceAttributeMappingService,
|
||||||
|
TaxonomyService,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/sources/{source_id}/categories")
|
||||||
|
def list_source_categories(source_id: str, session: Session = Depends(get_session)):
|
||||||
|
return {"items": CatalogService(session).source_categories(source_id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/sources/{source_id}/categories/{category}/discover-taxonomy", status_code=201)
|
||||||
|
def discover_taxonomy(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||||
|
return TaxonomyService(session).discover(source_id, category)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/sources/{source_id}/categories/{category}/filters")
|
||||||
|
def list_filters(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||||
|
return TaxonomyService(session).filters(source_id, category)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/criterion-definitions", status_code=201)
|
||||||
|
def create_criterion_definition(payload: CriterionDefinitionIn, session: Session = Depends(get_session)):
|
||||||
|
return CriterionDefinitionService(session).create(payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/criterion-definitions")
|
||||||
|
def list_criterion_definitions(category: str | None = None, session: Session = Depends(get_session)):
|
||||||
|
return {"items": CriterionDefinitionService(session).list(category)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/criterion-mappings")
|
||||||
|
def list_mappings(source_category_id: UUID | None = None, session: Session = Depends(get_session)):
|
||||||
|
return {"items": CriterionMappingService(session).list(source_category_id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/criterion-mappings", status_code=201)
|
||||||
|
def create_mapping(payload: CriterionMappingIn, session: Session = Depends(get_session)):
|
||||||
|
return CriterionMappingService(session).create(payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/criterion-mappings/{mapping_id}")
|
||||||
|
def patch_mapping(mapping_id: UUID, payload: CriterionMappingPatch, session: Session = Depends(get_session)):
|
||||||
|
return CriterionMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/source-attribute-mappings")
|
||||||
|
def list_attribute_mappings(session: Session = Depends(get_session)):
|
||||||
|
return {"items": SourceAttributeMappingService(session).list()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/source-attribute-mappings", status_code=201)
|
||||||
|
def create_attribute_mapping(payload: SourceAttributeMappingIn, session: Session = Depends(get_session)):
|
||||||
|
return SourceAttributeMappingService(session).create(payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/source-attribute-mappings/{mapping_id}")
|
||||||
|
def patch_attribute_mapping(
|
||||||
|
mapping_id: UUID,
|
||||||
|
payload: SourceAttributeMappingPatch,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
):
|
||||||
|
return SourceAttributeMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/discovery-searches", status_code=201)
|
||||||
|
def create_search(payload: DiscoverySearchIn, session: Session = Depends(get_session)):
|
||||||
|
return DiscoverySearchService(session).create(payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/discovery-searches")
|
||||||
|
def list_searches(due: bool = Query(default=False), session: Session = Depends(get_session)):
|
||||||
|
service = DiscoverySearchService(session)
|
||||||
|
return {"items": service.list_due() if due else service.list_all()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/discovery-searches/{search_id}")
|
||||||
|
def get_search(search_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return DiscoverySearchService(session).get(search_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/discovery-searches/{search_id}")
|
||||||
|
def patch_search(search_id: UUID, payload: DiscoverySearchPatch, session: Session = Depends(get_session)):
|
||||||
|
return DiscoverySearchService(session).patch(search_id, payload.model_dump(exclude_unset=True))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/discovery-searches/{search_id}/generate-query")
|
||||||
|
def generate_query(search_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return DiscoverySearchService(session).generate_query(search_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/discovery-searches/{search_id}/run", status_code=201)
|
||||||
|
def run_search(search_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return DiscoverySearchService(session).run(search_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/discovery-searches/{search_id}/runs")
|
||||||
|
def list_runs(search_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return {"items": DiscoverySearchService(session).list_runs(search_id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/search-runs/{run_id}")
|
||||||
|
def get_run(run_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return DiscoverySearchService(session).get_run(run_id)
|
||||||
20
backend/app/api/evaluations.py
Normal file
20
backend/app/api/evaluations.py
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.schemas import EvaluateIn
|
||||||
|
from app.persistence.db import get_session
|
||||||
|
from app.services.evaluation_service import evaluate_and_store, load_evaluation
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/evaluate/{variant_id}", status_code=201)
|
||||||
|
def evaluate(variant_id: UUID, payload: EvaluateIn):
|
||||||
|
return evaluate_and_store(variant_id=variant_id, payload=payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/evaluations/{evaluation_id}")
|
||||||
|
def get_evaluation(evaluation_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return load_evaluation(session, evaluation_id)
|
||||||
42
backend/app/api/offers.py
Normal file
42
backend/app/api/offers.py
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Response
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.schemas import OfferIn, OfferOut, PriceIn, PriceOut, page, parse_page
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.persistence.db import get_session
|
||||||
|
from app.persistence.repositories import CatalogRepository
|
||||||
|
from app.services.ingestion_service import IngestionService
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/variants/{variant_id}/offers", status_code=201)
|
||||||
|
def create_offer(variant_id: UUID, payload: OfferIn, session: Session = Depends(get_session)) -> OfferOut:
|
||||||
|
offer = IngestionService(session).create_offer(variant_id, payload.model_dump())
|
||||||
|
return OfferOut.model_validate(offer.__dict__)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/offers/{offer_id}")
|
||||||
|
def get_offer(offer_id: UUID, session: Session = Depends(get_session)) -> OfferOut:
|
||||||
|
offer = CatalogRepository(session).get_offer(offer_id)
|
||||||
|
if offer is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
||||||
|
return OfferOut.model_validate(offer.__dict__)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/offers/{offer_id}/prices")
|
||||||
|
def add_price(offer_id: UUID, payload: PriceIn, response: Response, session: Session = Depends(get_session)) -> PriceOut:
|
||||||
|
observation, created = IngestionService(session).add_price(offer_id, payload.model_dump())
|
||||||
|
response.status_code = 201 if created else 200
|
||||||
|
return PriceOut.model_validate(observation.__dict__)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/offers/{offer_id}/prices")
|
||||||
|
def list_prices(offer_id: UUID, limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||||
|
if CatalogRepository(session).get_offer(offer_id) is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
||||||
|
limit, offset = parse_page(limit, offset)
|
||||||
|
items, total = CatalogRepository(session).list_observations(offer_id, limit, offset)
|
||||||
|
return page([PriceOut.model_validate(item.__dict__) for item in items], limit, offset, total)
|
||||||
63
backend/app/api/products.py
Normal file
63
backend/app/api/products.py
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.schemas import AttributeIn, AttributeOut, ProductIn, ProductOut, VariantIn, VariantOut
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.persistence.db import get_session
|
||||||
|
from app.persistence.repositories import CatalogRepository
|
||||||
|
from app.services.ingestion_service import IngestionService
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/products", status_code=201)
|
||||||
|
def create_product(payload: ProductIn, session: Session = Depends(get_session)) -> ProductOut:
|
||||||
|
product = IngestionService(session).create_product(payload.model_dump())
|
||||||
|
return ProductOut.model_validate(product.__dict__)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/products/{product_id}")
|
||||||
|
def get_product(product_id: UUID, session: Session = Depends(get_session)) -> ProductOut:
|
||||||
|
product = CatalogRepository(session).get_product(product_id)
|
||||||
|
if product is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
|
||||||
|
return ProductOut.model_validate(product.__dict__)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/products/{product_id}/variants", status_code=201)
|
||||||
|
def create_variant(product_id: UUID, payload: VariantIn, session: Session = Depends(get_session)):
|
||||||
|
variant = IngestionService(session).create_variant(product_id, payload.model_dump())
|
||||||
|
return {
|
||||||
|
"id": variant.id,
|
||||||
|
"product_id": variant.product_id,
|
||||||
|
"name": variant.name,
|
||||||
|
"manufacturer_sku": variant.manufacturer_sku,
|
||||||
|
"identifiers": variant.identifiers,
|
||||||
|
"created_at": variant.created_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/variants/{variant_id}")
|
||||||
|
def get_variant(variant_id: UUID, session: Session = Depends(get_session)) -> VariantOut:
|
||||||
|
repo = CatalogRepository(session)
|
||||||
|
variant = repo.get_variant(variant_id)
|
||||||
|
if variant is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||||
|
attributes = [AttributeOut.model_validate(item.__dict__) for item in repo.list_attributes(variant_id)]
|
||||||
|
return VariantOut(
|
||||||
|
id=variant.id,
|
||||||
|
product_id=variant.product_id,
|
||||||
|
name=variant.name,
|
||||||
|
manufacturer_sku=variant.manufacturer_sku,
|
||||||
|
identifiers=variant.identifiers,
|
||||||
|
created_at=variant.created_at,
|
||||||
|
attributes=attributes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/variants/{variant_id}/attributes", status_code=201)
|
||||||
|
def add_attribute(variant_id: UUID, payload: AttributeIn, session: Session = Depends(get_session)) -> AttributeOut:
|
||||||
|
statement = IngestionService(session).add_attribute(variant_id, payload.model_dump())
|
||||||
|
return AttributeOut.model_validate(statement.__dict__)
|
||||||
100
backend/app/api/profiles.py
Normal file
100
backend/app/api/profiles.py
Normal file
|
|
@ -0,0 +1,100 @@
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Body, Depends
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.schemas import AttributeDefinitionIn, page, parse_page
|
||||||
|
from app.domain.serialize import jsonable
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.persistence.db import get_session
|
||||||
|
from app.persistence.repositories import CatalogRepository
|
||||||
|
from app.services.ingestion_service import IngestionService
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/attribute-definitions", status_code=201)
|
||||||
|
def create_attribute_definition(payload: AttributeDefinitionIn, session: Session = Depends(get_session)):
|
||||||
|
definition = IngestionService(session).create_definition(payload.model_dump())
|
||||||
|
return jsonable(definition.__dict__)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/attribute-definitions")
|
||||||
|
def list_attribute_definitions(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||||
|
limit, offset = parse_page(limit, offset)
|
||||||
|
items, total = CatalogRepository(session).list_definitions(limit, offset)
|
||||||
|
return page([jsonable(item.__dict__) for item in items], limit, offset, total)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/sources")
|
||||||
|
def list_sources(session: Session = Depends(get_session)):
|
||||||
|
items = CatalogRepository(session).list_sources()
|
||||||
|
return {
|
||||||
|
"items": [{"id": item.id, "name": item.name, "adapter_type": item.adapter_type} for item in items],
|
||||||
|
"limit": len(items),
|
||||||
|
"offset": 0,
|
||||||
|
"total": len(items),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/scoring-models/{version}")
|
||||||
|
def get_scoring_model(version: str, session: Session = Depends(get_session)):
|
||||||
|
model = CatalogRepository(session).get_scoring_model(version)
|
||||||
|
if model is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "scoring model not found", {"version": version})
|
||||||
|
return {
|
||||||
|
"version": model.version,
|
||||||
|
"algorithm_id": model.algorithm_id,
|
||||||
|
"definition": model.definition,
|
||||||
|
"definition_hash": model.definition_hash,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/profiles", status_code=201)
|
||||||
|
def create_profile(payload: dict[str, Any] = Body(...), session: Session = Depends(get_session)):
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise AppError(422, "PROFILE_INVALID", "profile must be an object")
|
||||||
|
row = IngestionService(session).create_profile(payload)
|
||||||
|
return {
|
||||||
|
"id": row.id,
|
||||||
|
"version": row.version,
|
||||||
|
"name": row.name,
|
||||||
|
"category": row.category,
|
||||||
|
"scoring_version": row.scoring_version,
|
||||||
|
"definition": row.definition,
|
||||||
|
"definition_hash": row.definition_hash,
|
||||||
|
"created_at": row.created_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/profiles")
|
||||||
|
def list_profiles(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||||
|
limit, offset = parse_page(limit, offset)
|
||||||
|
items, total = CatalogRepository(session).list_profiles(limit, offset)
|
||||||
|
return page(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": row.id,
|
||||||
|
"version": row.version,
|
||||||
|
"name": row.name,
|
||||||
|
"category": row.category,
|
||||||
|
"scoring_version": row.scoring_version,
|
||||||
|
"definition_hash": row.definition_hash,
|
||||||
|
"created_at": row.created_at,
|
||||||
|
}
|
||||||
|
for row in items
|
||||||
|
],
|
||||||
|
limit,
|
||||||
|
offset,
|
||||||
|
total,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/profiles/{profile_id}")
|
||||||
|
def get_profile(profile_id: str, version: int | None = None, session: Session = Depends(get_session)):
|
||||||
|
if version is None:
|
||||||
|
raise AppError(422, "VERSION_REQUIRED", "query parameter version is required")
|
||||||
|
profile = CatalogRepository(session).get_profile(profile_id, version)
|
||||||
|
if profile is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "profile version not found")
|
||||||
|
return profile.definition
|
||||||
125
backend/app/api/projects.py
Normal file
125
backend/app/api/projects.py
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.schemas import (
|
||||||
|
BuyingProjectIn,
|
||||||
|
BuyingProjectPatch,
|
||||||
|
DiscoverySearchIn,
|
||||||
|
ProjectCriteriaIn,
|
||||||
|
page,
|
||||||
|
parse_page,
|
||||||
|
)
|
||||||
|
from app.persistence.db import get_session
|
||||||
|
from app.services.catalog_service import CatalogService
|
||||||
|
from app.services.discovery_service import CandidateRankingService, DiscoverySearchService
|
||||||
|
from app.services.project_service import ProjectService
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/product-categories")
|
||||||
|
def list_product_categories(session: Session = Depends(get_session)):
|
||||||
|
return {"items": CatalogService(session).list_product_categories()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/product-categories/{product_category_id}/sources")
|
||||||
|
def list_category_sources(product_category_id: str, session: Session = Depends(get_session)):
|
||||||
|
return {"items": CatalogService(session).sources_for_category(product_category_id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/product-categories/{product_category_id}/available-criteria")
|
||||||
|
def available_criteria_for_category(
|
||||||
|
product_category_id: str,
|
||||||
|
source: str = Query(...),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
):
|
||||||
|
return CatalogService(session).available_criteria(product_category_id, source)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-templates")
|
||||||
|
def list_buying_templates(session: Session = Depends(get_session)):
|
||||||
|
return {"items": CatalogService(session).list_templates()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/buying-projects", status_code=201)
|
||||||
|
def create_buying_project(payload: BuyingProjectIn, session: Session = Depends(get_session)):
|
||||||
|
return ProjectService(session).create(payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects")
|
||||||
|
def list_buying_projects(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||||
|
limit, offset = parse_page(limit, offset)
|
||||||
|
items, total = ProjectService(session).list(limit, offset)
|
||||||
|
return page(items, limit, offset, total)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}")
|
||||||
|
def get_buying_project(project_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return ProjectService(session).get(project_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/buying-projects/{project_id}")
|
||||||
|
def patch_buying_project(project_id: UUID, payload: BuyingProjectPatch, session: Session = Depends(get_session)):
|
||||||
|
return ProjectService(session).patch(project_id, payload.model_dump(exclude_unset=True))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/buying-projects/{project_id}/criteria", status_code=201)
|
||||||
|
def add_buying_project_criteria(
|
||||||
|
project_id: UUID,
|
||||||
|
payload: ProjectCriteriaIn,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
):
|
||||||
|
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/buying-projects/{project_id}/criteria")
|
||||||
|
def put_buying_project_criteria(
|
||||||
|
project_id: UUID,
|
||||||
|
payload: ProjectCriteriaIn,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
):
|
||||||
|
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}/criteria")
|
||||||
|
def get_buying_project_criteria(project_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return ProjectService(session).get_criteria(project_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}/available-criteria")
|
||||||
|
def available_criteria(project_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return ProjectService(session).available_criteria(project_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}/discovery-searches")
|
||||||
|
def list_project_searches(project_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return {"items": DiscoverySearchService(session).list_for_project(project_id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/buying-projects/{project_id}/discovery-searches", status_code=201)
|
||||||
|
def create_project_search(project_id: UUID, payload: DiscoverySearchIn, session: Session = Depends(get_session)):
|
||||||
|
body = payload.model_dump()
|
||||||
|
body["buying_project_id"] = project_id
|
||||||
|
return DiscoverySearchService(session).create(body)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}/candidates")
|
||||||
|
def list_candidates(project_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return {"items": CandidateRankingService(session).list_candidates(project_id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}")
|
||||||
|
def get_candidate(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return CandidateRankingService(session).get_candidate(project_id, candidate_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}/observations")
|
||||||
|
def list_candidate_observations(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return {"items": CandidateRankingService(session).list_observations(project_id, candidate_id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/buying-projects/{project_id}/shortlist")
|
||||||
|
def list_shortlist(project_id: UUID, session: Session = Depends(get_session)):
|
||||||
|
return {"items": CandidateRankingService(session).list_candidates(project_id, shortlisted_only=True)}
|
||||||
274
backend/app/api/schemas.py
Normal file
274
backend/app/api/schemas.py
Normal file
|
|
@ -0,0 +1,274 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, field_serializer
|
||||||
|
|
||||||
|
|
||||||
|
class StrictModel(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AttributeDefinitionIn(StrictModel):
|
||||||
|
key: str
|
||||||
|
value_type: str
|
||||||
|
canonical_unit: str | None = None
|
||||||
|
minimum: Decimal | None = None
|
||||||
|
maximum: Decimal | None = None
|
||||||
|
description: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class AttributeDefinitionOut(StrictModel):
|
||||||
|
key: str
|
||||||
|
value_type: str
|
||||||
|
canonical_unit: str | None
|
||||||
|
minimum: Decimal | None
|
||||||
|
maximum: Decimal | None
|
||||||
|
description: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class ProductIn(StrictModel):
|
||||||
|
id: UUID | None = None
|
||||||
|
category: str
|
||||||
|
brand: str
|
||||||
|
name: str
|
||||||
|
model: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ProductOut(StrictModel):
|
||||||
|
id: UUID
|
||||||
|
category: str
|
||||||
|
brand: str
|
||||||
|
name: str
|
||||||
|
model: str | None
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class VariantIn(StrictModel):
|
||||||
|
id: UUID | None = None
|
||||||
|
name: str
|
||||||
|
manufacturer_sku: str | None = None
|
||||||
|
identifiers: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class AttributeIn(StrictModel):
|
||||||
|
id: UUID | None = None
|
||||||
|
attribute: str
|
||||||
|
raw_value: Any
|
||||||
|
normalized_value: Any | None = None
|
||||||
|
unit: str | None = None
|
||||||
|
normalization_status: str
|
||||||
|
normalizer_version: str | None = None
|
||||||
|
confidence: Decimal
|
||||||
|
source_id: str
|
||||||
|
source_reference: str
|
||||||
|
observed_at: datetime
|
||||||
|
normalization_error: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class AttributeOut(StrictModel):
|
||||||
|
id: UUID
|
||||||
|
variant_id: UUID
|
||||||
|
attribute: str
|
||||||
|
raw_value: Any
|
||||||
|
normalized_value: Any | None
|
||||||
|
unit: str | None
|
||||||
|
normalization_status: str
|
||||||
|
normalizer_version: str
|
||||||
|
confidence: Decimal
|
||||||
|
source_id: str
|
||||||
|
source_reference: str
|
||||||
|
observed_at: datetime
|
||||||
|
ingested_at: datetime
|
||||||
|
normalization_error: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class VariantOut(StrictModel):
|
||||||
|
id: UUID
|
||||||
|
product_id: UUID
|
||||||
|
name: str
|
||||||
|
manufacturer_sku: str | None
|
||||||
|
identifiers: dict[str, Any]
|
||||||
|
created_at: datetime
|
||||||
|
attributes: list[AttributeOut]
|
||||||
|
|
||||||
|
|
||||||
|
class OfferIn(StrictModel):
|
||||||
|
id: UUID | None = None
|
||||||
|
source_id: str
|
||||||
|
external_id: str
|
||||||
|
seller: str
|
||||||
|
url: str | None = None
|
||||||
|
market: str
|
||||||
|
currency: str
|
||||||
|
condition: str
|
||||||
|
|
||||||
|
|
||||||
|
class OfferOut(StrictModel):
|
||||||
|
id: UUID
|
||||||
|
variant_id: UUID
|
||||||
|
source_id: str
|
||||||
|
external_id: str
|
||||||
|
seller: str
|
||||||
|
url: str | None
|
||||||
|
market: str
|
||||||
|
currency: str
|
||||||
|
condition: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class PriceIn(StrictModel):
|
||||||
|
id: UUID | None = None
|
||||||
|
source_id: str
|
||||||
|
external_observation_id: str
|
||||||
|
observed_at: datetime
|
||||||
|
item_price: Decimal
|
||||||
|
shipping_price: Decimal | None = None
|
||||||
|
currency: str
|
||||||
|
availability: str
|
||||||
|
raw_payload: Any
|
||||||
|
confidence: Decimal
|
||||||
|
|
||||||
|
@field_serializer("item_price", "shipping_price")
|
||||||
|
def serialize_money(self, value: Decimal | None) -> str | None:
|
||||||
|
return None if value is None else format(value, "f")
|
||||||
|
|
||||||
|
|
||||||
|
class PriceOut(StrictModel):
|
||||||
|
id: UUID
|
||||||
|
offer_id: UUID
|
||||||
|
source_id: str
|
||||||
|
external_observation_id: str
|
||||||
|
observed_at: datetime
|
||||||
|
ingested_at: datetime
|
||||||
|
item_price: Decimal
|
||||||
|
shipping_price: Decimal | None
|
||||||
|
currency: str
|
||||||
|
availability: str
|
||||||
|
raw_payload: Any
|
||||||
|
confidence: Decimal
|
||||||
|
|
||||||
|
@field_serializer("item_price", "shipping_price")
|
||||||
|
def serialize_money(self, value: Decimal | None) -> str | None:
|
||||||
|
return None if value is None else format(Decimal(value).quantize(Decimal("0.01")), "f")
|
||||||
|
|
||||||
|
|
||||||
|
class EvaluateIn(StrictModel):
|
||||||
|
profile_id: str
|
||||||
|
profile_version: int
|
||||||
|
offer_id: UUID
|
||||||
|
as_of: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class BuyingProjectIn(StrictModel):
|
||||||
|
id: UUID | None = None
|
||||||
|
name: str
|
||||||
|
product_category_id: str | None = None
|
||||||
|
market: str | None = None
|
||||||
|
currency: str | None = None
|
||||||
|
target_budget: Decimal | None = None
|
||||||
|
template_id: str | None = None
|
||||||
|
template_version: int | None = None
|
||||||
|
profile_id: str | None = None
|
||||||
|
profile_version: int | None = None
|
||||||
|
workspace_id: str | None = None
|
||||||
|
shortlist_size: int = 5
|
||||||
|
schedule_interval_minutes: int | None = None
|
||||||
|
criteria: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class BuyingProjectPatch(StrictModel):
|
||||||
|
name: str | None = None
|
||||||
|
target_budget: Decimal | None = None
|
||||||
|
shortlist_size: int | None = None
|
||||||
|
status: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ProjectCriteriaIn(StrictModel):
|
||||||
|
items: list[dict[str, Any]]
|
||||||
|
|
||||||
|
|
||||||
|
class CriterionDefinitionIn(StrictModel):
|
||||||
|
key: str
|
||||||
|
value_type: str
|
||||||
|
unit: str | None = None
|
||||||
|
semantic: str | None = None
|
||||||
|
category: str | None = None
|
||||||
|
description: str | None = None
|
||||||
|
origin: str | None = None
|
||||||
|
source: str | None = None
|
||||||
|
product_category_id: str | None = None
|
||||||
|
label: str | None = None
|
||||||
|
scoring_attribute_key: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DiscoverySearchIn(StrictModel):
|
||||||
|
buying_project_id: UUID | None = None
|
||||||
|
source_id: str
|
||||||
|
source_category_id: UUID
|
||||||
|
schedule_interval_minutes: int = 180
|
||||||
|
broad_result_threshold: int = 300
|
||||||
|
|
||||||
|
|
||||||
|
class DiscoverySearchPatch(StrictModel):
|
||||||
|
schedule_interval_minutes: int | None = None
|
||||||
|
status: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class SourceAttributeMappingIn(StrictModel):
|
||||||
|
source_id: str
|
||||||
|
source_category_id: UUID | None = None
|
||||||
|
raw_name: str
|
||||||
|
criterion_definition_id: UUID
|
||||||
|
status: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class SourceAttributeMappingPatch(StrictModel):
|
||||||
|
raw_name: str | None = None
|
||||||
|
criterion_definition_id: UUID | None = None
|
||||||
|
status: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class CriterionMappingIn(StrictModel):
|
||||||
|
criterion_definition_id: UUID | None = None
|
||||||
|
criterion_key: str | None = None
|
||||||
|
source_id: str
|
||||||
|
source_category_id: UUID
|
||||||
|
external_filter_key: str
|
||||||
|
source_value: str | None = None
|
||||||
|
encoding: dict[str, Any] = Field(default_factory=lambda: {"url_param": "xf"})
|
||||||
|
status: str | None = None
|
||||||
|
value_type: str | None = None
|
||||||
|
unit: str | None = None
|
||||||
|
semantic: str | None = None
|
||||||
|
description: str | None = None
|
||||||
|
operator: str | None = None
|
||||||
|
probe_value: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
class CriterionMappingPatch(StrictModel):
|
||||||
|
source_value: str | None = None
|
||||||
|
encoding: dict[str, Any] | None = None
|
||||||
|
status: str | None = None
|
||||||
|
probe_value: Any = None
|
||||||
|
operator: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class Page(StrictModel):
|
||||||
|
items: list[Any]
|
||||||
|
limit: int
|
||||||
|
offset: int
|
||||||
|
total: int
|
||||||
|
|
||||||
|
|
||||||
|
def page(items: list[Any], limit: int, offset: int, total: int) -> dict[str, Any]:
|
||||||
|
return {"items": items, "limit": limit, "offset": offset, "total": total}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_page(limit: int = 50, offset: int = 0) -> tuple[int, int]:
|
||||||
|
from app.errors import AppError
|
||||||
|
|
||||||
|
if limit < 1 or limit > 200 or offset < 0:
|
||||||
|
raise AppError(422, "PAGINATION", "limit must be 1-200 and offset >= 0")
|
||||||
|
return limit, offset
|
||||||
12
backend/app/clock.py
Normal file
12
backend/app/clock.py
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from app.settings import get_settings
|
||||||
|
|
||||||
|
|
||||||
|
def now_utc() -> datetime:
|
||||||
|
override = get_settings().clock_now
|
||||||
|
if override is not None:
|
||||||
|
if override.tzinfo is None:
|
||||||
|
return override.replace(tzinfo=timezone.utc)
|
||||||
|
return override.astimezone(timezone.utc)
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
38
backend/app/discovery/__init__.py
Normal file
38
backend/app/discovery/__init__.py
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
from app.discovery.criteria import parse_project_criteria
|
||||||
|
from app.discovery.diff import diff_run, next_lifecycle, observations_changed
|
||||||
|
from app.discovery.enums import (
|
||||||
|
CandidateStatus,
|
||||||
|
DiffStatus,
|
||||||
|
LifecycleStatus,
|
||||||
|
MappingStatus,
|
||||||
|
MatchStatus,
|
||||||
|
ProjectStatus,
|
||||||
|
RunStatus,
|
||||||
|
SearchStatus,
|
||||||
|
)
|
||||||
|
from app.discovery.identity import IdentityDecision, IdentitySignals, decide_identity, spec_fingerprint
|
||||||
|
from app.discovery.mapping import verify_roundtrip
|
||||||
|
from app.discovery.ranking import rank_candidates
|
||||||
|
from app.discovery.urlgen import generate_search_url
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CandidateStatus",
|
||||||
|
"DiffStatus",
|
||||||
|
"IdentityDecision",
|
||||||
|
"IdentitySignals",
|
||||||
|
"LifecycleStatus",
|
||||||
|
"MappingStatus",
|
||||||
|
"MatchStatus",
|
||||||
|
"ProjectStatus",
|
||||||
|
"RunStatus",
|
||||||
|
"SearchStatus",
|
||||||
|
"decide_identity",
|
||||||
|
"diff_run",
|
||||||
|
"generate_search_url",
|
||||||
|
"next_lifecycle",
|
||||||
|
"observations_changed",
|
||||||
|
"parse_project_criteria",
|
||||||
|
"rank_candidates",
|
||||||
|
"spec_fingerprint",
|
||||||
|
"verify_roundtrip",
|
||||||
|
]
|
||||||
77
backend/app/discovery/criteria.py
Normal file
77
backend/app/discovery/criteria.py
Normal file
|
|
@ -0,0 +1,77 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.domain.validation import DomainValidationError, canonical_json, sha256_text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_project_criteria(definition: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
items = definition.get("items")
|
||||||
|
if not isinstance(items, list) or not items:
|
||||||
|
raise DomainValidationError("criteria.items must be a non-empty list")
|
||||||
|
seen: set[str] = set()
|
||||||
|
canonical_items: list[dict[str, Any]] = []
|
||||||
|
for raw in items:
|
||||||
|
criterion_id = raw.get("criterion_definition_id")
|
||||||
|
key = raw.get("criterion")
|
||||||
|
item_id = str(raw.get("id") or criterion_id or key or "")
|
||||||
|
if not item_id:
|
||||||
|
raise DomainValidationError("each criterion needs id, criterion_definition_id or criterion")
|
||||||
|
if item_id in seen:
|
||||||
|
raise DomainValidationError("duplicate criterion id", {"id": item_id})
|
||||||
|
seen.add(item_id)
|
||||||
|
if not criterion_id and not key:
|
||||||
|
raise DomainValidationError("criterion_definition_id or criterion is required", {"id": item_id})
|
||||||
|
if not raw.get("operator"):
|
||||||
|
raise DomainValidationError("operator is required", {"id": item_id})
|
||||||
|
enabled = raw.get("enabled_for_discovery")
|
||||||
|
if enabled is None:
|
||||||
|
enabled = raw.get("mandatory", True)
|
||||||
|
canonical_items.append(
|
||||||
|
{
|
||||||
|
"id": item_id,
|
||||||
|
"criterion_definition_id": None if criterion_id is None else UUID(str(criterion_id)),
|
||||||
|
"criterion": None if key is None else str(key),
|
||||||
|
"operator": str(raw["operator"]),
|
||||||
|
"value": raw.get("value"),
|
||||||
|
"enabled_for_discovery": bool(enabled),
|
||||||
|
"mandatory": bool(enabled),
|
||||||
|
"value_type": raw.get("value_type"),
|
||||||
|
"unit": raw.get("unit"),
|
||||||
|
"semantic": raw.get("semantic"),
|
||||||
|
"category": raw.get("category"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
canonical = {
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": item["id"],
|
||||||
|
"criterion_definition_id": None
|
||||||
|
if item["criterion_definition_id"] is None
|
||||||
|
else str(item["criterion_definition_id"]),
|
||||||
|
"criterion": item["criterion"],
|
||||||
|
"operator": item["operator"],
|
||||||
|
"value": item["value"],
|
||||||
|
"enabled_for_discovery": item["enabled_for_discovery"],
|
||||||
|
}
|
||||||
|
for item in canonical_items
|
||||||
|
]
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"items": canonical_items,
|
||||||
|
"definition": canonical,
|
||||||
|
"definition_hash": sha256_text(canonical_json(canonical)),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def criterion_payload(row) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": row.item_id,
|
||||||
|
"criterion_definition_id": row.criterion_definition_id,
|
||||||
|
"criterion": row.criterion_key,
|
||||||
|
"operator": row.operator,
|
||||||
|
"value": row.value,
|
||||||
|
"enabled_for_discovery": row.enabled_for_discovery,
|
||||||
|
"mandatory": row.enabled_for_discovery,
|
||||||
|
}
|
||||||
51
backend/app/discovery/definitions.py
Normal file
51
backend/app/discovery/definitions.py
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.persistence.discovery_models import CriterionDefinitionRecord
|
||||||
|
|
||||||
|
|
||||||
|
def get_definition_by_key(session: Session, key: str) -> CriterionDefinitionRecord | None:
|
||||||
|
return session.scalars(select(CriterionDefinitionRecord).where(CriterionDefinitionRecord.key == key)).first()
|
||||||
|
|
||||||
|
|
||||||
|
def get_definition(session: Session, definition_id: UUID) -> CriterionDefinitionRecord | None:
|
||||||
|
return session.get(CriterionDefinitionRecord, definition_id)
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_definition(session: Session, payload: dict[str, Any]) -> CriterionDefinitionRecord:
|
||||||
|
key = str(payload["key"])
|
||||||
|
existing = get_definition_by_key(session, key)
|
||||||
|
if existing is not None:
|
||||||
|
if payload.get("label"):
|
||||||
|
existing.label = str(payload["label"])
|
||||||
|
if payload.get("scoring_attribute_key"):
|
||||||
|
existing.scoring_attribute_key = str(payload["scoring_attribute_key"])
|
||||||
|
if payload.get("product_category_id"):
|
||||||
|
existing.product_category_id = str(payload["product_category_id"])
|
||||||
|
if payload.get("origin"):
|
||||||
|
existing.origin = str(payload["origin"])
|
||||||
|
if payload.get("source"):
|
||||||
|
existing.source = str(payload["source"])
|
||||||
|
return existing
|
||||||
|
row = CriterionDefinitionRecord(
|
||||||
|
id=payload.get("id") or uuid4(),
|
||||||
|
key=key,
|
||||||
|
origin=str(payload.get("origin") or "SOURCE"),
|
||||||
|
source=payload.get("source"),
|
||||||
|
product_category_id=payload.get("product_category_id"),
|
||||||
|
label=str(payload.get("label") or key),
|
||||||
|
value_type=str(payload["value_type"]),
|
||||||
|
unit=payload.get("unit"),
|
||||||
|
semantic=payload.get("semantic"),
|
||||||
|
category=payload.get("category") or payload.get("product_category_id"),
|
||||||
|
description=payload.get("description"),
|
||||||
|
scoring_attribute_key=payload.get("scoring_attribute_key"),
|
||||||
|
)
|
||||||
|
session.add(row)
|
||||||
|
session.flush()
|
||||||
|
return row
|
||||||
74
backend/app/discovery/diff.py
Normal file
74
backend/app/discovery/diff.py
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.discovery.enums import DiffStatus, LifecycleStatus, RunStatus
|
||||||
|
|
||||||
|
|
||||||
|
RELEVANT_ATTR_IGNORE = {"title", "html", "presentation", "raw_title"}
|
||||||
|
|
||||||
|
|
||||||
|
def observations_changed(previous: dict[str, Any] | None, current: dict[str, Any] | None) -> bool:
|
||||||
|
if previous is None or current is None:
|
||||||
|
return previous != current
|
||||||
|
if _money(previous.get("best_price")) != _money(current.get("best_price")):
|
||||||
|
return True
|
||||||
|
if previous.get("availability") != current.get("availability"):
|
||||||
|
return True
|
||||||
|
if previous.get("seller_count") != current.get("seller_count"):
|
||||||
|
return True
|
||||||
|
prev_attrs = _technical(previous.get("normalized_attributes") or {})
|
||||||
|
curr_attrs = _technical(current.get("normalized_attributes") or {})
|
||||||
|
return prev_attrs != curr_attrs
|
||||||
|
|
||||||
|
|
||||||
|
def _money(value: Any) -> Decimal | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return Decimal(str(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _technical(attrs: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {key: attrs[key] for key in sorted(attrs) if key not in RELEVANT_ATTR_IGNORE}
|
||||||
|
|
||||||
|
|
||||||
|
def diff_run(
|
||||||
|
*,
|
||||||
|
previous_ids: set[Any],
|
||||||
|
current_ids: set[Any],
|
||||||
|
previous_obs: dict[Any, dict[str, Any]],
|
||||||
|
current_obs: dict[Any, dict[str, Any]],
|
||||||
|
run_status: str,
|
||||||
|
) -> dict[Any, DiffStatus]:
|
||||||
|
result: dict[Any, DiffStatus] = {}
|
||||||
|
for ref_id in current_ids:
|
||||||
|
if ref_id not in previous_ids:
|
||||||
|
result[ref_id] = DiffStatus.NEW
|
||||||
|
elif observations_changed(previous_obs.get(ref_id), current_obs.get(ref_id)):
|
||||||
|
result[ref_id] = DiffStatus.CHANGED
|
||||||
|
else:
|
||||||
|
result[ref_id] = DiffStatus.UNCHANGED
|
||||||
|
if run_status == RunStatus.SUCCESS.value:
|
||||||
|
for ref_id in previous_ids - current_ids:
|
||||||
|
result[ref_id] = DiffStatus.MISSING
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def next_lifecycle(
|
||||||
|
*,
|
||||||
|
current: str,
|
||||||
|
missing_in_success_run: bool,
|
||||||
|
consecutive_missing: int,
|
||||||
|
reappeared: bool,
|
||||||
|
) -> tuple[str, int]:
|
||||||
|
if current == LifecycleStatus.ARCHIVED.value:
|
||||||
|
return current, consecutive_missing
|
||||||
|
if reappeared:
|
||||||
|
return LifecycleStatus.ACTIVE.value, 0
|
||||||
|
if missing_in_success_run:
|
||||||
|
streak = consecutive_missing + 1
|
||||||
|
if streak >= 3:
|
||||||
|
return LifecycleStatus.DISAPPEARED.value, streak
|
||||||
|
return LifecycleStatus.ACTIVE.value, streak
|
||||||
|
return current, consecutive_missing
|
||||||
78
backend/app/discovery/enums.py
Normal file
78
backend/app/discovery/enums.py
Normal file
|
|
@ -0,0 +1,78 @@
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
|
||||||
|
class ProjectStatus(StrEnum):
|
||||||
|
ACTIVE = "ACTIVE"
|
||||||
|
PURCHASED = "PURCHASED"
|
||||||
|
ARCHIVED = "ARCHIVED"
|
||||||
|
|
||||||
|
|
||||||
|
class MappingStatus(StrEnum):
|
||||||
|
DISCOVERED = "DISCOVERED"
|
||||||
|
LEARNED_UNVERIFIED = "LEARNED_UNVERIFIED"
|
||||||
|
VERIFIED = "VERIFIED"
|
||||||
|
BROKEN = "BROKEN"
|
||||||
|
DISABLED = "DISABLED"
|
||||||
|
|
||||||
|
|
||||||
|
class FilterType(StrEnum):
|
||||||
|
BOOLEAN = "BOOLEAN"
|
||||||
|
ENUM = "ENUM"
|
||||||
|
MULTI_ENUM = "MULTI_ENUM"
|
||||||
|
NUMERIC_RANGE = "NUMERIC_RANGE"
|
||||||
|
NUMERIC_THRESHOLD = "NUMERIC_THRESHOLD"
|
||||||
|
TEXT = "TEXT"
|
||||||
|
UNKNOWN = "UNKNOWN"
|
||||||
|
|
||||||
|
|
||||||
|
class CriterionOrigin(StrEnum):
|
||||||
|
SOURCE = "SOURCE"
|
||||||
|
INTERNAL = "INTERNAL"
|
||||||
|
|
||||||
|
|
||||||
|
class SearchStatus(StrEnum):
|
||||||
|
ACTIVE = "ACTIVE"
|
||||||
|
OUT_OF_SYNC = "OUT_OF_SYNC"
|
||||||
|
BLOCKED = "BLOCKED"
|
||||||
|
DISABLED = "DISABLED"
|
||||||
|
TOO_BROAD = "TOO_BROAD"
|
||||||
|
PAUSED = "PAUSED"
|
||||||
|
|
||||||
|
|
||||||
|
class RunStatus(StrEnum):
|
||||||
|
RUNNING = "RUNNING"
|
||||||
|
SUCCESS = "SUCCESS"
|
||||||
|
PARTIAL = "PARTIAL"
|
||||||
|
FAILED = "FAILED"
|
||||||
|
BLOCKED = "BLOCKED"
|
||||||
|
TOO_BROAD = "TOO_BROAD"
|
||||||
|
|
||||||
|
|
||||||
|
class RankingStatus(StrEnum):
|
||||||
|
NOT_CONFIGURED = "NOT_CONFIGURED"
|
||||||
|
CONFIGURED = "CONFIGURED"
|
||||||
|
|
||||||
|
|
||||||
|
class DiffStatus(StrEnum):
|
||||||
|
NEW = "NEW"
|
||||||
|
CHANGED = "CHANGED"
|
||||||
|
UNCHANGED = "UNCHANGED"
|
||||||
|
MISSING = "MISSING"
|
||||||
|
|
||||||
|
|
||||||
|
class LifecycleStatus(StrEnum):
|
||||||
|
ACTIVE = "ACTIVE"
|
||||||
|
DISAPPEARED = "DISAPPEARED"
|
||||||
|
ARCHIVED = "ARCHIVED"
|
||||||
|
|
||||||
|
|
||||||
|
class MatchStatus(StrEnum):
|
||||||
|
MATCHED = "MATCHED"
|
||||||
|
MATCH_UNCERTAIN = "MATCH_UNCERTAIN"
|
||||||
|
UNCERTAIN = "MATCH_UNCERTAIN"
|
||||||
|
NEW_REFERENCE = "NEW_REFERENCE"
|
||||||
|
|
||||||
|
|
||||||
|
class CandidateStatus(StrEnum):
|
||||||
|
CANDIDATE = "CANDIDATE"
|
||||||
|
SHORTLISTED = "SHORTLISTED"
|
||||||
81
backend/app/discovery/identity.py
Normal file
81
backend/app/discovery/identity.py
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.discovery.enums import MatchStatus
|
||||||
|
|
||||||
|
PRESENTATION_KEYS = {"title", "html", "presentation", "raw_title"}
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_title(title: str) -> str:
|
||||||
|
return re.sub(r"\s+", " ", title.casefold().strip())
|
||||||
|
|
||||||
|
|
||||||
|
def spec_fingerprint(normalized_attributes: dict[str, Any], keys: tuple[str, ...] | None = None) -> str:
|
||||||
|
"""Fingerprint from whatever technical keys the observation actually has."""
|
||||||
|
use = keys if keys is not None else tuple(sorted(key for key in normalized_attributes if key not in PRESENTATION_KEYS))
|
||||||
|
parts = []
|
||||||
|
for key in use:
|
||||||
|
value = normalized_attributes.get(key)
|
||||||
|
parts.append(f"{key}={'' if value is None else str(value).casefold().strip()}")
|
||||||
|
return "|".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class IdentitySignals:
|
||||||
|
source_external_id: str | None
|
||||||
|
gtin_ean: str | None
|
||||||
|
manufacturer_sku: str | None
|
||||||
|
source_url: str | None
|
||||||
|
title: str
|
||||||
|
manufacturer: str | None
|
||||||
|
model: str | None
|
||||||
|
fingerprint: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class IdentityDecision:
|
||||||
|
match_status: MatchStatus
|
||||||
|
existing_id: Any | None
|
||||||
|
reason: str
|
||||||
|
|
||||||
|
|
||||||
|
def decide_identity(incoming: IdentitySignals, known: list[tuple[Any, IdentitySignals]]) -> IdentityDecision:
|
||||||
|
"""Conservative matching. Similar titles never merge by themselves."""
|
||||||
|
for ref_id, existing in known:
|
||||||
|
if incoming.source_external_id and incoming.source_external_id == existing.source_external_id:
|
||||||
|
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_external_id")
|
||||||
|
if incoming.gtin_ean and incoming.gtin_ean == existing.gtin_ean:
|
||||||
|
return IdentityDecision(MatchStatus.MATCHED, ref_id, "gtin_ean")
|
||||||
|
if incoming.manufacturer_sku and incoming.manufacturer_sku == existing.manufacturer_sku:
|
||||||
|
return IdentityDecision(MatchStatus.MATCHED, ref_id, "manufacturer_sku")
|
||||||
|
if incoming.source_url and incoming.source_url == existing.source_url:
|
||||||
|
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_url")
|
||||||
|
|
||||||
|
title_hits = [
|
||||||
|
(ref_id, existing)
|
||||||
|
for ref_id, existing in known
|
||||||
|
if normalize_title(incoming.title) == normalize_title(existing.title)
|
||||||
|
]
|
||||||
|
if len(title_hits) == 1 and incoming.fingerprint and incoming.fingerprint == title_hits[0][1].fingerprint:
|
||||||
|
return IdentityDecision(MatchStatus.MATCHED, title_hits[0][0], "title+fingerprint")
|
||||||
|
|
||||||
|
manufacturer_model = [
|
||||||
|
(ref_id, existing)
|
||||||
|
for ref_id, existing in known
|
||||||
|
if incoming.manufacturer
|
||||||
|
and incoming.model
|
||||||
|
and incoming.manufacturer.casefold() == (existing.manufacturer or "").casefold()
|
||||||
|
and incoming.model.casefold() == (existing.model or "").casefold()
|
||||||
|
]
|
||||||
|
if len(manufacturer_model) == 1:
|
||||||
|
existing = manufacturer_model[0][1]
|
||||||
|
if incoming.fingerprint and existing.fingerprint and incoming.fingerprint != existing.fingerprint:
|
||||||
|
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "manufacturer+model conflict with specs")
|
||||||
|
return IdentityDecision(MatchStatus.MATCHED, manufacturer_model[0][0], "manufacturer+model")
|
||||||
|
|
||||||
|
if title_hits:
|
||||||
|
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "title only")
|
||||||
|
return IdentityDecision(MatchStatus.NEW_REFERENCE, None, "no match")
|
||||||
34
backend/app/discovery/mapping.py
Normal file
34
backend/app/discovery/mapping.py
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import parse_qsl, urlsplit
|
||||||
|
|
||||||
|
from app.discovery.enums import MappingStatus
|
||||||
|
from app.discovery.urlgen import decode_xf_roundtrip, generate_search_url
|
||||||
|
|
||||||
|
|
||||||
|
def verify_roundtrip(
|
||||||
|
*,
|
||||||
|
category_url: str,
|
||||||
|
criterion: dict[str, Any],
|
||||||
|
mapping: dict[str, Any],
|
||||||
|
) -> bool:
|
||||||
|
"""True when encoding a single mapping round-trips to the same source parameter."""
|
||||||
|
probe = {**mapping, "status": MappingStatus.VERIFIED.value}
|
||||||
|
url, internal = generate_search_url(
|
||||||
|
category_url=category_url,
|
||||||
|
criteria_items=[criterion],
|
||||||
|
mappings=[probe],
|
||||||
|
)
|
||||||
|
if internal:
|
||||||
|
return False
|
||||||
|
encoding = mapping.get("encoding") or {}
|
||||||
|
param = str(encoding.get("url_param") or "xf")
|
||||||
|
if param == "xf":
|
||||||
|
expected_key = str(mapping["external_filter_key"])
|
||||||
|
expected_value = str(mapping.get("source_value") or "")
|
||||||
|
decoded = decode_xf_roundtrip(url)
|
||||||
|
return any(key == expected_key and value == expected_value.replace("+", " ") for key, value in decoded)
|
||||||
|
expected = str(criterion.get("value"))
|
||||||
|
query = parse_qsl(urlsplit(url).query, keep_blank_values=True)
|
||||||
|
return any(name == param and value == expected for name, value in query)
|
||||||
100
backend/app/discovery/ranking.py
Normal file
100
backend/app/discovery/ranking.py
Normal file
|
|
@ -0,0 +1,100 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from app.clock import now_utc
|
||||||
|
from app.domain.attributes import select_attribute_snapshot
|
||||||
|
from app.domain.constraints import evaluate_constraint
|
||||||
|
from app.domain.enums import Eligibility, NormalizationStatus
|
||||||
|
from app.domain.models import AttributeStatement, Product
|
||||||
|
from app.scoring.engine import _product_eligibility
|
||||||
|
from app.scoring.mathutil import round_score
|
||||||
|
from app.scoring.product import score_product
|
||||||
|
|
||||||
|
|
||||||
|
def rank_candidates(
|
||||||
|
*,
|
||||||
|
profile,
|
||||||
|
scoring_model,
|
||||||
|
definitions,
|
||||||
|
candidates: list[dict[str, Any]],
|
||||||
|
shortlist_size: int,
|
||||||
|
as_of: datetime | None = None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
scored: list[dict[str, Any]] = []
|
||||||
|
when = as_of or now_utc()
|
||||||
|
for candidate in candidates:
|
||||||
|
attrs = _statements(candidate, when)
|
||||||
|
snapshot = select_attribute_snapshot(
|
||||||
|
attrs,
|
||||||
|
as_of=when,
|
||||||
|
snapshot_cutoff=when,
|
||||||
|
)
|
||||||
|
constraint_results = tuple(
|
||||||
|
evaluate_constraint(
|
||||||
|
constraint,
|
||||||
|
snapshot.get(constraint.attribute),
|
||||||
|
definitions[constraint.attribute],
|
||||||
|
scoring_model.minimum_confidence,
|
||||||
|
)
|
||||||
|
for constraint in profile.constraints
|
||||||
|
if constraint.attribute in definitions
|
||||||
|
)
|
||||||
|
product = Product(
|
||||||
|
id=uuid4(),
|
||||||
|
category=profile.category,
|
||||||
|
brand=str(candidate.get("manufacturer") or "unknown"),
|
||||||
|
name=str(candidate.get("source_title") or candidate.get("name") or "candidate"),
|
||||||
|
model=candidate.get("model"),
|
||||||
|
created_at=when,
|
||||||
|
)
|
||||||
|
_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
|
||||||
|
product_score, coverage, _components = score_product(
|
||||||
|
profile.preferences,
|
||||||
|
snapshot,
|
||||||
|
constraint_results,
|
||||||
|
scoring_model,
|
||||||
|
)
|
||||||
|
scored.append(
|
||||||
|
{
|
||||||
|
**candidate,
|
||||||
|
"product_eligibility": product_eligibility.value,
|
||||||
|
"product_score": round_score(product_score),
|
||||||
|
"product_coverage": coverage,
|
||||||
|
"eligible": product_eligibility == Eligibility.ELIGIBLE,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
scored.sort(key=lambda item: (not item["eligible"], -(item["product_score"] or Decimal("0"))))
|
||||||
|
for index, item in enumerate(scored):
|
||||||
|
item["candidate_status"] = "SHORTLISTED" if index < shortlist_size and item["eligible"] else "CANDIDATE"
|
||||||
|
item["rank"] = index + 1
|
||||||
|
return scored
|
||||||
|
|
||||||
|
|
||||||
|
def _statements(candidate: dict[str, Any], when: datetime) -> tuple[AttributeStatement, ...]:
|
||||||
|
raw = candidate.get("normalized_attributes") or {}
|
||||||
|
now = when
|
||||||
|
items = []
|
||||||
|
variant_id = candidate.get("variant_id") or uuid4()
|
||||||
|
for key, value in raw.items():
|
||||||
|
items.append(
|
||||||
|
AttributeStatement(
|
||||||
|
id=uuid4(),
|
||||||
|
variant_id=variant_id if isinstance(variant_id, UUID) else uuid4(),
|
||||||
|
attribute=key,
|
||||||
|
raw_value=value,
|
||||||
|
normalized_value=value,
|
||||||
|
unit=None,
|
||||||
|
normalization_status=NormalizationStatus.NORMALIZED,
|
||||||
|
normalizer_version="discovery-v1",
|
||||||
|
confidence=Decimal("1"),
|
||||||
|
source_id="geizhals",
|
||||||
|
source_reference="discovery",
|
||||||
|
observed_at=now,
|
||||||
|
ingested_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(items)
|
||||||
82
backend/app/discovery/urlgen.py
Normal file
82
backend/app/discovery/urlgen.py
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlsplit, urlunsplit
|
||||||
|
|
||||||
|
from app.discovery.enums import MappingStatus
|
||||||
|
|
||||||
|
|
||||||
|
def generate_search_url(
|
||||||
|
*,
|
||||||
|
category_url: str,
|
||||||
|
criteria_items: list[dict[str, Any]],
|
||||||
|
mappings: list[dict[str, Any]],
|
||||||
|
) -> tuple[str, list[str]]:
|
||||||
|
"""Build a source URL from VERIFIED mappings only. INTERNAL_ONLY keys are skipped."""
|
||||||
|
by_criterion = {item["criterion_key"]: item for item in mappings if item.get("status") == MappingStatus.VERIFIED.value}
|
||||||
|
fragments: list[tuple[str, str]] = []
|
||||||
|
internal_only: list[str] = []
|
||||||
|
for criterion in criteria_items:
|
||||||
|
key = str(criterion["criterion"])
|
||||||
|
mapping = by_criterion.get(key)
|
||||||
|
if mapping is None:
|
||||||
|
internal_only.append(key)
|
||||||
|
continue
|
||||||
|
fragments.extend(_encode_mapping(mapping, criterion))
|
||||||
|
fragments.sort()
|
||||||
|
base = urlsplit(category_url)
|
||||||
|
query = dict(parse_qsl(base.query, keep_blank_values=True))
|
||||||
|
extra: list[tuple[str, str]] = []
|
||||||
|
for name, value in fragments:
|
||||||
|
if name in query and name != "xf":
|
||||||
|
query[name] = value
|
||||||
|
else:
|
||||||
|
extra.append((name, value))
|
||||||
|
encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote)
|
||||||
|
url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment))
|
||||||
|
return url, internal_only
|
||||||
|
|
||||||
|
|
||||||
|
def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||||
|
encoding = mapping.get("encoding") or {}
|
||||||
|
param = str(encoding.get("url_param") or "xf")
|
||||||
|
if param == "xf":
|
||||||
|
filter_key = str(mapping["external_filter_key"])
|
||||||
|
source_value = str(mapping.get("source_value") or _threshold_value(mapping, criterion))
|
||||||
|
return [("xf", f"{filter_key}_{_plus(source_value)}")]
|
||||||
|
if param in {"minp", "maxp", "prmin", "prmax"}:
|
||||||
|
return [(param, str(criterion.get("value")))]
|
||||||
|
template = encoding.get("value_template")
|
||||||
|
if template:
|
||||||
|
return [(param, str(template).format(value=criterion.get("value"), **mapping))]
|
||||||
|
raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}")
|
||||||
|
|
||||||
|
|
||||||
|
def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
||||||
|
option = mapping.get("source_value")
|
||||||
|
if option:
|
||||||
|
return str(option)
|
||||||
|
value = criterion.get("value")
|
||||||
|
unit = mapping.get("unit") or ""
|
||||||
|
operator = criterion.get("operator")
|
||||||
|
if operator in {">=", ">"} and unit:
|
||||||
|
return f"ab {value}{unit}"
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _plus(value: str) -> str:
|
||||||
|
return value.replace(" ", "+")
|
||||||
|
|
||||||
|
|
||||||
|
def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]:
|
||||||
|
parts = urlsplit(url)
|
||||||
|
decoded: list[tuple[str, str]] = []
|
||||||
|
for name, value in parse_qsl(parts.query, keep_blank_values=True):
|
||||||
|
if name == "xf" and "_" in value:
|
||||||
|
key, raw = value.split("_", 1)
|
||||||
|
decoded.append((key, raw.replace("+", " ")))
|
||||||
|
return decoded
|
||||||
|
|
||||||
|
|
||||||
|
def same_origin_url(base: str, maybe_relative: str) -> str:
|
||||||
|
return urljoin(base, maybe_relative)
|
||||||
0
backend/app/domain/__init__.py
Normal file
0
backend/app/domain/__init__.py
Normal file
45
backend/app/domain/attributes.py
Normal file
45
backend/app/domain/attributes.py
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
from app.domain.enums import NormalizationStatus
|
||||||
|
from app.domain.models import AttributeStatement
|
||||||
|
|
||||||
|
|
||||||
|
def statement_is_usable(statement: AttributeStatement | None, minimum_confidence: Decimal) -> bool:
|
||||||
|
if statement is None:
|
||||||
|
return False
|
||||||
|
if statement.normalization_status != NormalizationStatus.NORMALIZED:
|
||||||
|
return False
|
||||||
|
if statement.normalized_value is None:
|
||||||
|
return False
|
||||||
|
return statement.confidence >= minimum_confidence
|
||||||
|
|
||||||
|
|
||||||
|
def select_attribute_snapshot(
|
||||||
|
statements: Iterable[AttributeStatement],
|
||||||
|
*,
|
||||||
|
as_of: datetime,
|
||||||
|
snapshot_cutoff: datetime,
|
||||||
|
) -> dict[str, AttributeStatement]:
|
||||||
|
"""Pick the newest statement per key at or before the evaluation cutoff."""
|
||||||
|
chosen: dict[str, AttributeStatement] = {}
|
||||||
|
for statement in statements:
|
||||||
|
if statement.observed_at > as_of or statement.ingested_at > snapshot_cutoff:
|
||||||
|
continue
|
||||||
|
current = chosen.get(statement.attribute)
|
||||||
|
if current is None or _is_newer(statement, current):
|
||||||
|
chosen[statement.attribute] = statement
|
||||||
|
return chosen
|
||||||
|
|
||||||
|
|
||||||
|
def _is_newer(candidate: AttributeStatement, current: AttributeStatement) -> bool:
|
||||||
|
return (
|
||||||
|
candidate.observed_at,
|
||||||
|
candidate.ingested_at,
|
||||||
|
candidate.id,
|
||||||
|
) > (
|
||||||
|
current.observed_at,
|
||||||
|
current.ingested_at,
|
||||||
|
current.id,
|
||||||
|
)
|
||||||
148
backend/app/domain/constraints.py
Normal file
148
backend/app/domain/constraints.py
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.domain.attributes import statement_is_usable
|
||||||
|
from app.domain.enums import ConstraintOperator, ConstraintStatus, ValueType
|
||||||
|
from app.domain.models import AttributeDefinition, AttributeStatement, Constraint, ConstraintResult
|
||||||
|
|
||||||
|
_NUMBER_OPS = {
|
||||||
|
ConstraintOperator.GE,
|
||||||
|
ConstraintOperator.LE,
|
||||||
|
ConstraintOperator.GT,
|
||||||
|
ConstraintOperator.LT,
|
||||||
|
}
|
||||||
|
_EQUALITY_OPS = {ConstraintOperator.EQ, ConstraintOperator.NE}
|
||||||
|
_SCALAR_TYPES = {ValueType.NUMBER, ValueType.STRING, ValueType.BOOLEAN}
|
||||||
|
|
||||||
|
|
||||||
|
def operator_allowed_for(value_type: ValueType, operator: ConstraintOperator) -> bool:
|
||||||
|
if operator == ConstraintOperator.EXISTS:
|
||||||
|
return True
|
||||||
|
if operator in _NUMBER_OPS:
|
||||||
|
return value_type == ValueType.NUMBER
|
||||||
|
if operator in _EQUALITY_OPS:
|
||||||
|
return value_type in _SCALAR_TYPES
|
||||||
|
if operator == ConstraintOperator.IN:
|
||||||
|
return value_type in _SCALAR_TYPES
|
||||||
|
if operator == ConstraintOperator.CONTAINS:
|
||||||
|
return value_type == ValueType.STRING_LIST
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_constraint(
|
||||||
|
constraint: Constraint,
|
||||||
|
statement: AttributeStatement | None,
|
||||||
|
definition: AttributeDefinition,
|
||||||
|
minimum_confidence: Decimal,
|
||||||
|
) -> ConstraintResult:
|
||||||
|
usable = statement_is_usable(statement, minimum_confidence)
|
||||||
|
actual = statement.normalized_value if usable else None
|
||||||
|
attribute_row_id = statement.id if statement is not None else None
|
||||||
|
|
||||||
|
if not usable:
|
||||||
|
return ConstraintResult(
|
||||||
|
constraint_id=constraint.id,
|
||||||
|
attribute=constraint.attribute,
|
||||||
|
operator=constraint.operator.value,
|
||||||
|
expected=constraint.value,
|
||||||
|
actual=None,
|
||||||
|
status=ConstraintStatus.UNKNOWN.value,
|
||||||
|
mandatory=constraint.mandatory,
|
||||||
|
attribute_row_id=attribute_row_id,
|
||||||
|
reason_code="ATTRIBUTE_UNUSABLE",
|
||||||
|
)
|
||||||
|
|
||||||
|
passed = _compare(constraint.operator, actual, constraint.value, definition.value_type)
|
||||||
|
if passed is None:
|
||||||
|
status = ConstraintStatus.UNKNOWN
|
||||||
|
reason = "TYPE_MISMATCH"
|
||||||
|
elif passed:
|
||||||
|
status = ConstraintStatus.PASS
|
||||||
|
reason = "CONSTRAINT_PASS"
|
||||||
|
else:
|
||||||
|
status = ConstraintStatus.FAIL
|
||||||
|
reason = "CONSTRAINT_FAIL"
|
||||||
|
return ConstraintResult(
|
||||||
|
constraint_id=constraint.id,
|
||||||
|
attribute=constraint.attribute,
|
||||||
|
operator=constraint.operator.value,
|
||||||
|
expected=constraint.value,
|
||||||
|
actual=actual,
|
||||||
|
status=status.value,
|
||||||
|
mandatory=constraint.mandatory,
|
||||||
|
attribute_row_id=attribute_row_id,
|
||||||
|
reason_code=reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _as_number(value: Any) -> Decimal | None:
|
||||||
|
if isinstance(value, bool) or value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, Decimal):
|
||||||
|
return value
|
||||||
|
if isinstance(value, int):
|
||||||
|
return Decimal(value)
|
||||||
|
if isinstance(value, float):
|
||||||
|
return Decimal(str(value))
|
||||||
|
if isinstance(value, str):
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _same_scalar(left: Any, right: Any, value_type: ValueType) -> bool | None:
|
||||||
|
if value_type == ValueType.NUMBER:
|
||||||
|
left_n = _as_number(left)
|
||||||
|
right_n = _as_number(right)
|
||||||
|
if left_n is None or right_n is None:
|
||||||
|
return None
|
||||||
|
return left_n == right_n
|
||||||
|
if value_type == ValueType.STRING:
|
||||||
|
if not isinstance(left, str) or not isinstance(right, str):
|
||||||
|
return None
|
||||||
|
return left == right
|
||||||
|
if value_type == ValueType.BOOLEAN:
|
||||||
|
if not isinstance(left, bool) or not isinstance(right, bool):
|
||||||
|
return None
|
||||||
|
return left == right
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _compare(
|
||||||
|
operator: ConstraintOperator,
|
||||||
|
actual: Any,
|
||||||
|
expected: Any,
|
||||||
|
value_type: ValueType,
|
||||||
|
) -> bool | None:
|
||||||
|
if operator == ConstraintOperator.EXISTS:
|
||||||
|
return True
|
||||||
|
if operator in _NUMBER_OPS:
|
||||||
|
actual_n = _as_number(actual)
|
||||||
|
expected_n = _as_number(expected)
|
||||||
|
if actual_n is None or expected_n is None:
|
||||||
|
return None
|
||||||
|
if operator == ConstraintOperator.GE:
|
||||||
|
return actual_n >= expected_n
|
||||||
|
if operator == ConstraintOperator.LE:
|
||||||
|
return actual_n <= expected_n
|
||||||
|
if operator == ConstraintOperator.GT:
|
||||||
|
return actual_n > expected_n
|
||||||
|
return actual_n < expected_n
|
||||||
|
if operator == ConstraintOperator.EQ:
|
||||||
|
return _same_scalar(actual, expected, value_type)
|
||||||
|
if operator == ConstraintOperator.NE:
|
||||||
|
equal = _same_scalar(actual, expected, value_type)
|
||||||
|
return None if equal is None else (not equal)
|
||||||
|
if operator == ConstraintOperator.IN:
|
||||||
|
if not isinstance(expected, list) or not expected:
|
||||||
|
return None
|
||||||
|
matches = [_same_scalar(actual, item, value_type) for item in expected]
|
||||||
|
if any(item is None for item in matches):
|
||||||
|
return None
|
||||||
|
return any(matches)
|
||||||
|
if operator == ConstraintOperator.CONTAINS:
|
||||||
|
if not isinstance(actual, list) or not isinstance(expected, str):
|
||||||
|
return None
|
||||||
|
if any(not isinstance(item, str) for item in actual):
|
||||||
|
return None
|
||||||
|
return expected in actual
|
||||||
|
return None
|
||||||
81
backend/app/domain/enums.py
Normal file
81
backend/app/domain/enums.py
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
|
||||||
|
class ValueType(StrEnum):
|
||||||
|
NUMBER = "number"
|
||||||
|
STRING = "string"
|
||||||
|
BOOLEAN = "boolean"
|
||||||
|
STRING_LIST = "string_list"
|
||||||
|
|
||||||
|
|
||||||
|
class OfferCondition(StrEnum):
|
||||||
|
NEW = "new"
|
||||||
|
REFURBISHED = "refurbished"
|
||||||
|
USED = "used"
|
||||||
|
|
||||||
|
|
||||||
|
class Availability(StrEnum):
|
||||||
|
IN_STOCK = "in_stock"
|
||||||
|
OUT_OF_STOCK = "out_of_stock"
|
||||||
|
UNKNOWN = "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
class NormalizationStatus(StrEnum):
|
||||||
|
NORMALIZED = "normalized"
|
||||||
|
UNKNOWN = "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
class ConstraintStatus(StrEnum):
|
||||||
|
PASS = "PASS"
|
||||||
|
FAIL = "FAIL"
|
||||||
|
UNKNOWN = "UNKNOWN"
|
||||||
|
|
||||||
|
|
||||||
|
class Eligibility(StrEnum):
|
||||||
|
ELIGIBLE = "ELIGIBLE"
|
||||||
|
INELIGIBLE = "INELIGIBLE"
|
||||||
|
UNKNOWN = "UNKNOWN"
|
||||||
|
|
||||||
|
|
||||||
|
class PreferenceStrategy(StrEnum):
|
||||||
|
NUMERIC_LINEAR = "numeric_linear"
|
||||||
|
|
||||||
|
|
||||||
|
class PreferenceDirection(StrEnum):
|
||||||
|
HIGHER = "higher"
|
||||||
|
LOWER = "lower"
|
||||||
|
|
||||||
|
|
||||||
|
class ConstraintOperator(StrEnum):
|
||||||
|
GE = ">="
|
||||||
|
LE = "<="
|
||||||
|
GT = ">"
|
||||||
|
LT = "<"
|
||||||
|
EQ = "=="
|
||||||
|
NE = "!="
|
||||||
|
IN = "in"
|
||||||
|
CONTAINS = "contains"
|
||||||
|
EXISTS = "exists"
|
||||||
|
|
||||||
|
|
||||||
|
class Classification(StrEnum):
|
||||||
|
IGNORE = "IGNORE"
|
||||||
|
WATCH = "WATCH"
|
||||||
|
GOOD = "GOOD"
|
||||||
|
VERY_GOOD = "VERY_GOOD"
|
||||||
|
EXCEPTIONAL = "EXCEPTIONAL"
|
||||||
|
INELIGIBLE = "INELIGIBLE"
|
||||||
|
INSUFFICIENT_DATA = "INSUFFICIENT_DATA"
|
||||||
|
|
||||||
|
|
||||||
|
class PriceBasis(StrEnum):
|
||||||
|
BUDGET_ONLY = "BUDGET_ONLY"
|
||||||
|
BUDGET_AND_HISTORY = "BUDGET_AND_HISTORY"
|
||||||
|
|
||||||
|
|
||||||
|
class PriceReferenceScope(StrEnum):
|
||||||
|
OFFER = "offer"
|
||||||
|
|
||||||
|
|
||||||
|
class AdapterType(StrEnum):
|
||||||
|
MANUAL = "manual"
|
||||||
275
backend/app/domain/models.py
Normal file
275
backend/app/domain/models.py
Normal file
|
|
@ -0,0 +1,275 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.domain.enums import (
|
||||||
|
Availability,
|
||||||
|
ConstraintOperator,
|
||||||
|
Eligibility,
|
||||||
|
NormalizationStatus,
|
||||||
|
OfferCondition,
|
||||||
|
PreferenceDirection,
|
||||||
|
PreferenceStrategy,
|
||||||
|
PriceBasis,
|
||||||
|
PriceReferenceScope,
|
||||||
|
ValueType,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Budget:
|
||||||
|
target: Decimal
|
||||||
|
maximum: Decimal
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Constraint:
|
||||||
|
id: str
|
||||||
|
attribute: str
|
||||||
|
operator: ConstraintOperator
|
||||||
|
value: Any = None
|
||||||
|
mandatory: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Preference:
|
||||||
|
id: str
|
||||||
|
attribute: str
|
||||||
|
strategy: PreferenceStrategy
|
||||||
|
direction: PreferenceDirection
|
||||||
|
acceptable: Decimal
|
||||||
|
ideal: Decimal
|
||||||
|
weight: Decimal
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ClassificationBound:
|
||||||
|
upper: Decimal
|
||||||
|
inclusive: bool
|
||||||
|
label: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ScoringModel:
|
||||||
|
version: str
|
||||||
|
algorithm_id: str
|
||||||
|
supported_preference_strategies: tuple[str, ...]
|
||||||
|
minimum_confidence: Decimal
|
||||||
|
maximum_price_age_hours: int
|
||||||
|
optional_constraint_penalty_points: Decimal
|
||||||
|
missing_preference_score: Decimal
|
||||||
|
product_weight: Decimal
|
||||||
|
price_weight: Decimal
|
||||||
|
budget_weight: Decimal
|
||||||
|
history_weight: Decimal
|
||||||
|
history_window_days: int
|
||||||
|
minimum_history_days: int
|
||||||
|
history_discount_for_max_score: Decimal
|
||||||
|
classification_upper_bounds: tuple[ClassificationBound, ...]
|
||||||
|
definition: dict[str, Any]
|
||||||
|
definition_hash: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Profile:
|
||||||
|
id: str
|
||||||
|
version: int
|
||||||
|
name: str
|
||||||
|
category: str
|
||||||
|
currency: str
|
||||||
|
market: str
|
||||||
|
allowed_conditions: tuple[OfferCondition, ...]
|
||||||
|
budget: Budget
|
||||||
|
scoring_version: str
|
||||||
|
constraints: tuple[Constraint, ...]
|
||||||
|
preferences: tuple[Preference, ...]
|
||||||
|
definition: dict[str, Any]
|
||||||
|
definition_hash: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AttributeDefinition:
|
||||||
|
key: str
|
||||||
|
value_type: ValueType
|
||||||
|
canonical_unit: str | None
|
||||||
|
minimum: Decimal | None
|
||||||
|
maximum: Decimal | None
|
||||||
|
description: str | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Product:
|
||||||
|
id: UUID
|
||||||
|
category: str
|
||||||
|
brand: str
|
||||||
|
name: str
|
||||||
|
model: str | None
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ProductVariant:
|
||||||
|
id: UUID
|
||||||
|
product_id: UUID
|
||||||
|
name: str
|
||||||
|
manufacturer_sku: str | None
|
||||||
|
identifiers: dict[str, Any]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AttributeStatement:
|
||||||
|
id: UUID
|
||||||
|
variant_id: UUID
|
||||||
|
attribute: str
|
||||||
|
raw_value: Any
|
||||||
|
normalized_value: Any
|
||||||
|
unit: str | None
|
||||||
|
normalization_status: NormalizationStatus
|
||||||
|
normalizer_version: str
|
||||||
|
confidence: Decimal
|
||||||
|
source_id: str
|
||||||
|
source_reference: str
|
||||||
|
observed_at: datetime
|
||||||
|
ingested_at: datetime
|
||||||
|
normalization_error: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Offer:
|
||||||
|
id: UUID
|
||||||
|
variant_id: UUID
|
||||||
|
source_id: str
|
||||||
|
external_id: str
|
||||||
|
seller: str
|
||||||
|
url: str | None
|
||||||
|
market: str
|
||||||
|
currency: str
|
||||||
|
condition: OfferCondition
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PriceObservation:
|
||||||
|
id: UUID
|
||||||
|
offer_id: UUID
|
||||||
|
source_id: str
|
||||||
|
external_observation_id: str
|
||||||
|
observed_at: datetime
|
||||||
|
ingested_at: datetime
|
||||||
|
item_price: Decimal
|
||||||
|
shipping_price: Decimal | None
|
||||||
|
currency: str
|
||||||
|
availability: Availability
|
||||||
|
raw_payload: Any
|
||||||
|
confidence: Decimal
|
||||||
|
|
||||||
|
@property
|
||||||
|
def total_price(self) -> Decimal | None:
|
||||||
|
if self.shipping_price is None:
|
||||||
|
return None
|
||||||
|
return self.item_price + self.shipping_price
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ConstraintResult:
|
||||||
|
constraint_id: str
|
||||||
|
attribute: str
|
||||||
|
operator: str
|
||||||
|
expected: Any
|
||||||
|
actual: Any
|
||||||
|
status: str
|
||||||
|
mandatory: bool
|
||||||
|
attribute_row_id: UUID | None
|
||||||
|
reason_code: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GateResult:
|
||||||
|
key: str
|
||||||
|
status: str
|
||||||
|
reason_code: str
|
||||||
|
details: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ScoreComponent:
|
||||||
|
kind: str
|
||||||
|
component_key: str
|
||||||
|
raw_score: Decimal | None
|
||||||
|
weight: Decimal | None
|
||||||
|
contribution: Decimal | None
|
||||||
|
details: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class HistoryWindow:
|
||||||
|
days: int
|
||||||
|
values: tuple[Decimal, ...]
|
||||||
|
observation_ids: tuple[UUID, ...]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def day_count(self) -> int:
|
||||||
|
return len(self.values)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PriceStatistics:
|
||||||
|
current_total: Decimal | None
|
||||||
|
median_30d: Decimal | None
|
||||||
|
median_30d_days: int
|
||||||
|
median_90d: Decimal | None
|
||||||
|
median_90d_days: int
|
||||||
|
lowest_365d: Decimal | None
|
||||||
|
lowest_365d_days: int
|
||||||
|
lowest_365d_is_full_year: bool
|
||||||
|
used_observation_ids: tuple[UUID, ...]
|
||||||
|
current_observation_id: UUID | None
|
||||||
|
price_reference_scope: PriceReferenceScope = PriceReferenceScope.OFFER
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Reason:
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class EvaluationResult:
|
||||||
|
profile_id: str
|
||||||
|
profile_version: int
|
||||||
|
scoring_version: str
|
||||||
|
scoring_hash: str
|
||||||
|
engine_build: str
|
||||||
|
variant_id: UUID
|
||||||
|
offer_id: UUID
|
||||||
|
as_of: datetime
|
||||||
|
snapshot_cutoff: datetime
|
||||||
|
product_eligibility: Eligibility
|
||||||
|
offer_eligibility: Eligibility
|
||||||
|
overall_eligibility: Eligibility
|
||||||
|
eligible: bool | None
|
||||||
|
product_score: Decimal | None
|
||||||
|
product_score_raw: Decimal | None
|
||||||
|
price_score: Decimal | None
|
||||||
|
price_score_raw: Decimal | None
|
||||||
|
deal_score: Decimal | None
|
||||||
|
deal_score_raw: Decimal | None
|
||||||
|
classification: str
|
||||||
|
price_basis: PriceBasis | None
|
||||||
|
product_coverage: Decimal
|
||||||
|
constraint_results: tuple[ConstraintResult, ...]
|
||||||
|
product_gates: tuple[GateResult, ...]
|
||||||
|
offer_gates: tuple[GateResult, ...]
|
||||||
|
components: tuple[ScoreComponent, ...]
|
||||||
|
price_statistics: PriceStatistics
|
||||||
|
price_reference_scope: PriceReferenceScope
|
||||||
|
reasons: tuple[Reason, ...]
|
||||||
|
warnings: tuple[str, ...]
|
||||||
|
input_snapshot: dict[str, Any]
|
||||||
|
failed_constraints: tuple[ConstraintResult, ...]
|
||||||
|
unknown_constraints: tuple[ConstraintResult, ...]
|
||||||
26
backend/app/domain/serialize.py
Normal file
26
backend/app/domain/serialize.py
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
from datetime import date, datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.domain.timeutil import to_iso_z
|
||||||
|
|
||||||
|
|
||||||
|
def jsonable(value: Any) -> Any:
|
||||||
|
if value is None or isinstance(value, (str, int, bool)):
|
||||||
|
return value
|
||||||
|
if isinstance(value, Decimal):
|
||||||
|
return format(value, "f")
|
||||||
|
if isinstance(value, UUID):
|
||||||
|
return str(value)
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return to_iso_z(value)
|
||||||
|
if isinstance(value, date):
|
||||||
|
return value.isoformat()
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {str(key): jsonable(item) for key, item in value.items()}
|
||||||
|
if isinstance(value, (list, tuple)):
|
||||||
|
return [jsonable(item) for item in value]
|
||||||
|
if hasattr(value, "value"):
|
||||||
|
return jsonable(value.value)
|
||||||
|
return value
|
||||||
11
backend/app/domain/timeutil.py
Normal file
11
backend/app/domain/timeutil.py
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
|
||||||
|
def require_utc(value: datetime, field_name: str) -> datetime:
|
||||||
|
if value.tzinfo is None:
|
||||||
|
raise ValueError(f"{field_name} must include a timezone")
|
||||||
|
return value.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def to_iso_z(value: datetime) -> str:
|
||||||
|
return require_utc(value, "datetime").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
307
backend/app/domain/validation.py
Normal file
307
backend/app/domain/validation.py
Normal file
|
|
@ -0,0 +1,307 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.domain.constraints import operator_allowed_for
|
||||||
|
from app.domain.enums import (
|
||||||
|
ConstraintOperator,
|
||||||
|
OfferCondition,
|
||||||
|
PreferenceDirection,
|
||||||
|
PreferenceStrategy,
|
||||||
|
ValueType,
|
||||||
|
)
|
||||||
|
from app.domain.models import (
|
||||||
|
AttributeDefinition,
|
||||||
|
Budget,
|
||||||
|
ClassificationBound,
|
||||||
|
Constraint,
|
||||||
|
Preference,
|
||||||
|
Profile,
|
||||||
|
ScoringModel,
|
||||||
|
)
|
||||||
|
|
||||||
|
SNAKE_CASE = r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$"
|
||||||
|
CURRENCY = r"^[A-Z]{3}$"
|
||||||
|
SUPPORTED_ALGORITHM = "linear-budget-history-v1"
|
||||||
|
|
||||||
|
|
||||||
|
class DomainValidationError(ValueError):
|
||||||
|
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.details = details or {}
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_json(data: Any) -> str:
|
||||||
|
return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_text(text: str) -> str:
|
||||||
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _decimal(value: Any, field: str) -> Decimal:
|
||||||
|
if isinstance(value, bool) or value is None:
|
||||||
|
raise DomainValidationError(f"{field} must be a decimal", {"field": field})
|
||||||
|
try:
|
||||||
|
return Decimal(str(value))
|
||||||
|
except Exception as exc:
|
||||||
|
raise DomainValidationError(f"{field} must be a decimal", {"field": field}) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _require(mapping: dict[str, Any], key: str) -> Any:
|
||||||
|
if key not in mapping:
|
||||||
|
raise DomainValidationError(f"missing field {key}", {"field": key})
|
||||||
|
return mapping[key]
|
||||||
|
|
||||||
|
|
||||||
|
def validate_attribute_key(key: str) -> None:
|
||||||
|
import re
|
||||||
|
|
||||||
|
if not re.fullmatch(SNAKE_CASE, key):
|
||||||
|
raise DomainValidationError("attribute key must be snake_case", {"key": key})
|
||||||
|
|
||||||
|
|
||||||
|
def validate_currency(value: str) -> str:
|
||||||
|
import re
|
||||||
|
|
||||||
|
if not re.fullmatch(CURRENCY, value):
|
||||||
|
raise DomainValidationError("currency must be a 3-letter uppercase code", {"currency": value})
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def parse_scoring_model(definition: dict[str, Any]) -> ScoringModel:
|
||||||
|
version = str(_require(definition, "version"))
|
||||||
|
algorithm_id = str(_require(definition, "algorithm_id"))
|
||||||
|
if algorithm_id != SUPPORTED_ALGORITHM:
|
||||||
|
raise DomainValidationError("unsupported algorithm_id", {"algorithm_id": algorithm_id})
|
||||||
|
strategies = tuple(str(item) for item in _require(definition, "supported_preference_strategies"))
|
||||||
|
if not strategies:
|
||||||
|
raise DomainValidationError("supported_preference_strategies must not be empty")
|
||||||
|
if PreferenceStrategy.NUMERIC_LINEAR.value not in strategies:
|
||||||
|
raise DomainValidationError("generic-v1 must support numeric_linear")
|
||||||
|
|
||||||
|
min_conf = _decimal(_require(definition, "minimum_confidence"), "minimum_confidence")
|
||||||
|
if not Decimal("0") <= min_conf <= Decimal("1"):
|
||||||
|
raise DomainValidationError("minimum_confidence must be between 0 and 1")
|
||||||
|
|
||||||
|
max_age = int(_require(definition, "maximum_price_age_hours"))
|
||||||
|
history_window = int(_require(definition, "history_window_days"))
|
||||||
|
min_history = int(_require(definition, "minimum_history_days"))
|
||||||
|
if max_age <= 0 or history_window <= 0 or min_history <= 0:
|
||||||
|
raise DomainValidationError("time windows must be positive")
|
||||||
|
|
||||||
|
product_weight = _decimal(_require(definition, "product_weight"), "product_weight")
|
||||||
|
price_weight = _decimal(_require(definition, "price_weight"), "price_weight")
|
||||||
|
budget_weight = _decimal(_require(definition, "budget_weight"), "budget_weight")
|
||||||
|
history_weight = _decimal(_require(definition, "history_weight"), "history_weight")
|
||||||
|
for name, weight in (
|
||||||
|
("product_weight", product_weight),
|
||||||
|
("price_weight", price_weight),
|
||||||
|
("budget_weight", budget_weight),
|
||||||
|
("history_weight", history_weight),
|
||||||
|
):
|
||||||
|
if weight < 0:
|
||||||
|
raise DomainValidationError(f"{name} must be non-negative")
|
||||||
|
if product_weight + price_weight != Decimal("1"):
|
||||||
|
raise DomainValidationError("product_weight + price_weight must equal 1")
|
||||||
|
if budget_weight + history_weight != Decimal("1"):
|
||||||
|
raise DomainValidationError("budget_weight + history_weight must equal 1")
|
||||||
|
|
||||||
|
bounds_raw = _require(definition, "classification_upper_bounds")
|
||||||
|
bounds: list[ClassificationBound] = []
|
||||||
|
previous = Decimal("0")
|
||||||
|
for index, raw in enumerate(bounds_raw):
|
||||||
|
upper = _decimal(raw["upper"], f"classification_upper_bounds[{index}].upper")
|
||||||
|
inclusive = bool(raw["inclusive"])
|
||||||
|
label = str(raw["label"])
|
||||||
|
if upper < previous or (index and upper == previous):
|
||||||
|
raise DomainValidationError("classification bounds must be strictly ascending")
|
||||||
|
if index == 0 and upper <= 0:
|
||||||
|
raise DomainValidationError("classification bounds must start above 0")
|
||||||
|
bounds.append(ClassificationBound(upper=upper, inclusive=inclusive, label=label))
|
||||||
|
previous = upper
|
||||||
|
if not bounds or bounds[-1].upper != Decimal("100"):
|
||||||
|
raise DomainValidationError("classification bounds must end at 100")
|
||||||
|
|
||||||
|
canonical = json.loads(canonical_json(definition))
|
||||||
|
return ScoringModel(
|
||||||
|
version=version,
|
||||||
|
algorithm_id=algorithm_id,
|
||||||
|
supported_preference_strategies=strategies,
|
||||||
|
minimum_confidence=min_conf,
|
||||||
|
maximum_price_age_hours=max_age,
|
||||||
|
optional_constraint_penalty_points=_decimal(
|
||||||
|
_require(definition, "optional_constraint_penalty_points"),
|
||||||
|
"optional_constraint_penalty_points",
|
||||||
|
),
|
||||||
|
missing_preference_score=_decimal(
|
||||||
|
_require(definition, "missing_preference_score"),
|
||||||
|
"missing_preference_score",
|
||||||
|
),
|
||||||
|
product_weight=product_weight,
|
||||||
|
price_weight=price_weight,
|
||||||
|
budget_weight=budget_weight,
|
||||||
|
history_weight=history_weight,
|
||||||
|
history_window_days=history_window,
|
||||||
|
minimum_history_days=min_history,
|
||||||
|
history_discount_for_max_score=_decimal(
|
||||||
|
_require(definition, "history_discount_for_max_score"),
|
||||||
|
"history_discount_for_max_score",
|
||||||
|
),
|
||||||
|
classification_upper_bounds=tuple(bounds),
|
||||||
|
definition=canonical,
|
||||||
|
definition_hash=sha256_text(canonical_json(canonical)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_profile(
|
||||||
|
definition: dict[str, Any],
|
||||||
|
*,
|
||||||
|
attribute_definitions: dict[str, AttributeDefinition],
|
||||||
|
scoring_versions: set[str],
|
||||||
|
supported_strategies: set[str],
|
||||||
|
) -> Profile:
|
||||||
|
profile_id = str(_require(definition, "id"))
|
||||||
|
version = int(_require(definition, "version"))
|
||||||
|
if version <= 0:
|
||||||
|
raise DomainValidationError("profile version must be > 0")
|
||||||
|
name = str(_require(definition, "name"))
|
||||||
|
category = str(_require(definition, "category"))
|
||||||
|
if not category:
|
||||||
|
raise DomainValidationError("category is required")
|
||||||
|
currency = validate_currency(str(_require(definition, "currency")))
|
||||||
|
market = str(_require(definition, "market"))
|
||||||
|
if not market:
|
||||||
|
raise DomainValidationError("market is required")
|
||||||
|
raw_conditions = _require(definition, "allowed_conditions")
|
||||||
|
if not raw_conditions:
|
||||||
|
raise DomainValidationError("allowed_conditions is required")
|
||||||
|
conditions = tuple(OfferCondition(item) for item in raw_conditions)
|
||||||
|
scoring_version = str(_require(definition, "scoring_version"))
|
||||||
|
if scoring_version not in scoring_versions:
|
||||||
|
raise DomainValidationError("unknown scoring_version", {"scoring_version": scoring_version})
|
||||||
|
|
||||||
|
budget_raw = _require(definition, "budget")
|
||||||
|
target = _decimal(_require(budget_raw, "target"), "budget.target")
|
||||||
|
maximum = _decimal(_require(budget_raw, "maximum"), "budget.maximum")
|
||||||
|
if not (Decimal("0") < target < maximum):
|
||||||
|
raise DomainValidationError("budget must satisfy 0 < target < maximum")
|
||||||
|
|
||||||
|
constraints = _parse_constraints(_require(definition, "constraints"), attribute_definitions)
|
||||||
|
preferences = _parse_preferences(
|
||||||
|
_require(definition, "preferences"),
|
||||||
|
attribute_definitions,
|
||||||
|
supported_strategies,
|
||||||
|
)
|
||||||
|
canonical = json.loads(canonical_json(definition))
|
||||||
|
return Profile(
|
||||||
|
id=profile_id,
|
||||||
|
version=version,
|
||||||
|
name=name,
|
||||||
|
category=category,
|
||||||
|
currency=currency,
|
||||||
|
market=market,
|
||||||
|
allowed_conditions=conditions,
|
||||||
|
budget=Budget(target=target, maximum=maximum),
|
||||||
|
scoring_version=scoring_version,
|
||||||
|
constraints=constraints,
|
||||||
|
preferences=preferences,
|
||||||
|
definition=canonical,
|
||||||
|
definition_hash=sha256_text(canonical_json(canonical)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_constraints(
|
||||||
|
raw_items: list[dict[str, Any]],
|
||||||
|
attribute_definitions: dict[str, AttributeDefinition],
|
||||||
|
) -> tuple[Constraint, ...]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
items: list[Constraint] = []
|
||||||
|
for raw in raw_items:
|
||||||
|
constraint_id = str(_require(raw, "id"))
|
||||||
|
if constraint_id in seen:
|
||||||
|
raise DomainValidationError("duplicate constraint id", {"id": constraint_id})
|
||||||
|
seen.add(constraint_id)
|
||||||
|
attribute = str(_require(raw, "attribute"))
|
||||||
|
definition = attribute_definitions.get(attribute)
|
||||||
|
if definition is None:
|
||||||
|
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
|
||||||
|
operator = ConstraintOperator(str(_require(raw, "operator")))
|
||||||
|
if not operator_allowed_for(definition.value_type, operator):
|
||||||
|
raise DomainValidationError(
|
||||||
|
"operator is not valid for attribute type",
|
||||||
|
{"attribute": attribute, "operator": operator.value, "value_type": definition.value_type.value},
|
||||||
|
)
|
||||||
|
value = raw.get("value")
|
||||||
|
if operator == ConstraintOperator.EXISTS:
|
||||||
|
if "value" in raw and raw["value"] is not None:
|
||||||
|
raise DomainValidationError("exists must not have a value", {"id": constraint_id})
|
||||||
|
value = None
|
||||||
|
elif operator == ConstraintOperator.IN:
|
||||||
|
if not isinstance(value, list) or not value:
|
||||||
|
raise DomainValidationError("in requires a non-empty list", {"id": constraint_id})
|
||||||
|
elif operator == ConstraintOperator.CONTAINS:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise DomainValidationError("contains requires a string", {"id": constraint_id})
|
||||||
|
items.append(
|
||||||
|
Constraint(
|
||||||
|
id=constraint_id,
|
||||||
|
attribute=attribute,
|
||||||
|
operator=operator,
|
||||||
|
value=value,
|
||||||
|
mandatory=bool(raw.get("mandatory", True)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(items)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_preferences(
|
||||||
|
raw_items: list[dict[str, Any]],
|
||||||
|
attribute_definitions: dict[str, AttributeDefinition],
|
||||||
|
supported_strategies: set[str],
|
||||||
|
) -> tuple[Preference, ...]:
|
||||||
|
if not raw_items:
|
||||||
|
raise DomainValidationError("at least one preference is required")
|
||||||
|
seen: set[str] = set()
|
||||||
|
items: list[Preference] = []
|
||||||
|
for raw in raw_items:
|
||||||
|
preference_id = str(_require(raw, "id"))
|
||||||
|
if preference_id in seen:
|
||||||
|
raise DomainValidationError("duplicate preference id", {"id": preference_id})
|
||||||
|
seen.add(preference_id)
|
||||||
|
attribute = str(_require(raw, "attribute"))
|
||||||
|
definition = attribute_definitions.get(attribute)
|
||||||
|
if definition is None:
|
||||||
|
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
|
||||||
|
strategy = str(_require(raw, "strategy"))
|
||||||
|
if strategy not in supported_strategies:
|
||||||
|
raise DomainValidationError("unsupported preference strategy", {"strategy": strategy})
|
||||||
|
if strategy != PreferenceStrategy.NUMERIC_LINEAR.value:
|
||||||
|
raise DomainValidationError("v0.1 only supports numeric_linear", {"strategy": strategy})
|
||||||
|
if definition.value_type != ValueType.NUMBER:
|
||||||
|
raise DomainValidationError("numeric_linear requires a number attribute", {"attribute": attribute})
|
||||||
|
direction = PreferenceDirection(str(_require(raw, "direction")))
|
||||||
|
acceptable = _decimal(_require(raw, "acceptable"), "acceptable")
|
||||||
|
ideal = _decimal(_require(raw, "ideal"), "ideal")
|
||||||
|
weight = _decimal(_require(raw, "weight"), "weight")
|
||||||
|
if weight <= 0:
|
||||||
|
raise DomainValidationError("preference weight must be positive", {"id": preference_id})
|
||||||
|
if direction == PreferenceDirection.HIGHER and not ideal > acceptable:
|
||||||
|
raise DomainValidationError("higher requires ideal > acceptable", {"id": preference_id})
|
||||||
|
if direction == PreferenceDirection.LOWER and not ideal < acceptable:
|
||||||
|
raise DomainValidationError("lower requires ideal < acceptable", {"id": preference_id})
|
||||||
|
items.append(
|
||||||
|
Preference(
|
||||||
|
id=preference_id,
|
||||||
|
attribute=attribute,
|
||||||
|
strategy=PreferenceStrategy.NUMERIC_LINEAR,
|
||||||
|
direction=direction,
|
||||||
|
acceptable=acceptable,
|
||||||
|
ideal=ideal,
|
||||||
|
weight=weight,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(items)
|
||||||
20
backend/app/errors.py
Normal file
20
backend/app/errors.py
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
class AppError(Exception):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
status_code: int,
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
details: dict[str, Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.status_code = status_code
|
||||||
|
self.code = code
|
||||||
|
self.message = message
|
||||||
|
self.details = details or {}
|
||||||
|
|
||||||
|
|
||||||
|
def error_body(code: str, message: str, details: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||||
|
return {"error": {"code": code, "message": message, "details": details or {}}}
|
||||||
48
backend/app/main.py
Normal file
48
backend/app/main.py
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
from fastapi import FastAPI, Request
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
from app.api.discovery import router as discovery_router
|
||||||
|
from app.api.evaluations import router as evaluations_router
|
||||||
|
from app.api.health import router as health_router
|
||||||
|
from app.api.offers import router as offers_router
|
||||||
|
from app.api.products import router as products_router
|
||||||
|
from app.api.profiles import router as profiles_router
|
||||||
|
from app.api.projects import router as projects_router
|
||||||
|
from app.errors import AppError, error_body
|
||||||
|
from app.settings import get_settings
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
app = FastAPI(
|
||||||
|
title="Buying Agent",
|
||||||
|
version="0.1.0",
|
||||||
|
description="Generic buying and deal evaluation API. The backend is the system of record; no frontend or n8n is required.",
|
||||||
|
)
|
||||||
|
app.add_middleware(
|
||||||
|
CORSMiddleware,
|
||||||
|
allow_origins=settings.cors_origins,
|
||||||
|
allow_credentials=True,
|
||||||
|
allow_methods=["*"],
|
||||||
|
allow_headers=["*"],
|
||||||
|
)
|
||||||
|
app.include_router(health_router)
|
||||||
|
app.include_router(profiles_router)
|
||||||
|
app.include_router(products_router)
|
||||||
|
app.include_router(offers_router)
|
||||||
|
app.include_router(evaluations_router)
|
||||||
|
app.include_router(projects_router)
|
||||||
|
app.include_router(discovery_router)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(AppError)
|
||||||
|
def handle_app_error(_request: Request, exc: AppError) -> JSONResponse:
|
||||||
|
return JSONResponse(error_body(exc.code, exc.message, exc.details), status_code=exc.status_code)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(RequestValidationError)
|
||||||
|
def handle_validation_error(_request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||||
|
return JSONResponse(
|
||||||
|
error_body("VALIDATION_ERROR", "request validation failed", {"errors": exc.errors()}),
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
0
backend/app/normalization/__init__.py
Normal file
0
backend/app/normalization/__init__.py
Normal file
11
backend/app/normalization/base.py
Normal file
11
backend/app/normalization/base.py
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
from app.domain.models import AttributeDefinition, AttributeStatement
|
||||||
|
|
||||||
|
|
||||||
|
class Normalizer(Protocol):
|
||||||
|
version: str
|
||||||
|
|
||||||
|
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
|
||||||
|
"""Validate an already normalized statement. Must not parse free text."""
|
||||||
|
...
|
||||||
61
backend/app/normalization/manual.py
Normal file
61
backend/app/normalization/manual.py
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.domain.enums import NormalizationStatus, ValueType
|
||||||
|
from app.domain.models import AttributeDefinition, AttributeStatement
|
||||||
|
from app.errors import AppError
|
||||||
|
|
||||||
|
|
||||||
|
class ManualNormalizer:
|
||||||
|
version = "manual-v1"
|
||||||
|
|
||||||
|
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
|
||||||
|
if statement.normalization_status == NormalizationStatus.UNKNOWN:
|
||||||
|
if statement.normalized_value is not None:
|
||||||
|
raise AppError(
|
||||||
|
422,
|
||||||
|
"NORMALIZATION_INCONSISTENT",
|
||||||
|
"unknown statements must not have a normalized_value",
|
||||||
|
)
|
||||||
|
return statement
|
||||||
|
if statement.normalization_status != NormalizationStatus.NORMALIZED:
|
||||||
|
raise AppError(422, "NORMALIZATION_STATUS", "unsupported normalization_status")
|
||||||
|
if statement.normalized_value is None:
|
||||||
|
raise AppError(422, "NORMALIZATION_MISSING", "normalized_value is required")
|
||||||
|
_assert_type(definition, statement.normalized_value)
|
||||||
|
if definition.value_type == ValueType.NUMBER:
|
||||||
|
number = Decimal(str(statement.normalized_value))
|
||||||
|
if definition.key == "weight_kg" and number <= 0:
|
||||||
|
raise AppError(422, "VALUE_RANGE", "weight_kg must be > 0")
|
||||||
|
if definition.minimum is not None and number < definition.minimum:
|
||||||
|
raise AppError(422, "VALUE_RANGE", f"{definition.key} is below the minimum")
|
||||||
|
if definition.maximum is not None and number > definition.maximum:
|
||||||
|
raise AppError(422, "VALUE_RANGE", f"{definition.key} is above the maximum")
|
||||||
|
if definition.canonical_unit and statement.unit != definition.canonical_unit:
|
||||||
|
raise AppError(
|
||||||
|
422,
|
||||||
|
"UNIT_MISMATCH",
|
||||||
|
"unit must match the canonical unit",
|
||||||
|
{"expected": definition.canonical_unit, "actual": statement.unit},
|
||||||
|
)
|
||||||
|
if definition.canonical_unit is None and statement.unit not in {None, ""}:
|
||||||
|
raise AppError(422, "UNIT_MISMATCH", "this attribute has no canonical unit")
|
||||||
|
return statement
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_type(definition: AttributeDefinition, value: Any) -> None:
|
||||||
|
if definition.value_type == ValueType.NUMBER:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, float, Decimal)):
|
||||||
|
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a number")
|
||||||
|
return
|
||||||
|
if definition.value_type == ValueType.STRING:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a string")
|
||||||
|
return
|
||||||
|
if definition.value_type == ValueType.BOOLEAN:
|
||||||
|
if not isinstance(value, bool):
|
||||||
|
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a boolean")
|
||||||
|
return
|
||||||
|
if definition.value_type == ValueType.STRING_LIST:
|
||||||
|
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
|
||||||
|
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a list of strings")
|
||||||
0
backend/app/persistence/__init__.py
Normal file
0
backend/app/persistence/__init__.py
Normal file
|
|
@ -14,5 +14,9 @@ def get_session() -> Generator[Session, None, None]:
|
||||||
session = SessionLocal()
|
session = SessionLocal()
|
||||||
try:
|
try:
|
||||||
yield session
|
yield session
|
||||||
|
session.commit()
|
||||||
|
except Exception:
|
||||||
|
session.rollback()
|
||||||
|
raise
|
||||||
finally:
|
finally:
|
||||||
session.close()
|
session.close()
|
||||||
321
backend/app/persistence/discovery_models.py
Normal file
321
backend/app/persistence/discovery_models.py
Normal file
|
|
@ -0,0 +1,321 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import Boolean, DateTime, ForeignKey, ForeignKeyConstraint, Integer, Numeric, String, Text, UniqueConstraint
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.persistence.models import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ProductCategoryRecord(Base):
|
||||||
|
__tablename__ = "product_categories"
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False, default="ACTIVE")
|
||||||
|
|
||||||
|
|
||||||
|
class BuyingProjectRecord(Base):
|
||||||
|
__tablename__ = "buying_projects"
|
||||||
|
__table_args__ = (
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["template_id", "template_version"],
|
||||||
|
["product_profiles.id", "product_profiles.version"],
|
||||||
|
name="fk_buying_projects_template",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
workspace_id: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
product_category_id: Mapped[str] = mapped_column(ForeignKey("product_categories.id"), nullable=False)
|
||||||
|
market: Mapped[str] = mapped_column(String(8), nullable=False)
|
||||||
|
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
||||||
|
target_budget: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||||
|
template_id: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
template_version: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
shortlist_size: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
|
||||||
|
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
archived_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def profile_id(self) -> str | None:
|
||||||
|
return self.template_id
|
||||||
|
|
||||||
|
@property
|
||||||
|
def profile_version(self) -> int | None:
|
||||||
|
return self.template_version
|
||||||
|
|
||||||
|
|
||||||
|
class BuyingProjectCriteriaRecord(Base):
|
||||||
|
__tablename__ = "buying_project_criteria"
|
||||||
|
|
||||||
|
project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), primary_key=True)
|
||||||
|
version: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ProjectCriterionRecord(Base):
|
||||||
|
__tablename__ = "project_criteria"
|
||||||
|
__table_args__ = (
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["project_id", "version"],
|
||||||
|
["buying_project_criteria.project_id", "buying_project_criteria.version"],
|
||||||
|
name="fk_project_criteria_version",
|
||||||
|
),
|
||||||
|
UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
project_id: Mapped[UUID] = mapped_column(nullable=False)
|
||||||
|
version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
item_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
||||||
|
criterion_key: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
operator: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
value: Mapped[Any] = mapped_column(JSONB)
|
||||||
|
enabled_for_discovery: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def mandatory(self) -> bool:
|
||||||
|
return self.enabled_for_discovery
|
||||||
|
|
||||||
|
@property
|
||||||
|
def criteria_version(self) -> int:
|
||||||
|
return self.version
|
||||||
|
|
||||||
|
|
||||||
|
class SourceCategoryRecord(Base):
|
||||||
|
__tablename__ = "source_categories"
|
||||||
|
__table_args__ = (UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
external_category_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
source_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
|
||||||
|
canonical_category: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
last_verified_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
|
||||||
|
class SourceFilterDefinitionRecord(Base):
|
||||||
|
__tablename__ = "source_filter_definitions"
|
||||||
|
__table_args__ = (UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||||
|
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
label: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
filter_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
canonical_unit: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class SourceFilterOptionRecord(Base):
|
||||||
|
__tablename__ = "source_filter_options"
|
||||||
|
__table_args__ = (UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), nullable=False)
|
||||||
|
source_value: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
||||||
|
operator: Mapped[str | None] = mapped_column(String(16))
|
||||||
|
unit: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
|
||||||
|
|
||||||
|
class SourceFilterRangeRecord(Base):
|
||||||
|
__tablename__ = "source_filter_ranges"
|
||||||
|
|
||||||
|
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), primary_key=True)
|
||||||
|
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||||
|
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||||
|
step: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||||
|
unit: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
|
||||||
|
|
||||||
|
class CriterionDefinitionRecord(Base):
|
||||||
|
__tablename__ = "criterion_definitions"
|
||||||
|
__table_args__ = (UniqueConstraint("key", name="uq_criterion_definition_key"),)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
origin: Mapped[str] = mapped_column(String(16), nullable=False, default="SOURCE")
|
||||||
|
source: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
|
||||||
|
label: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
unit: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
semantic: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
category: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
|
scoring_attribute_key: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
|
||||||
|
|
||||||
|
class SourceAttributeMappingRecord(Base):
|
||||||
|
__tablename__ = "source_attribute_mappings"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
source_category_id: Mapped[UUID | None] = mapped_column(ForeignKey("source_categories.id"))
|
||||||
|
raw_name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class CriterionMappingRecord(Base):
|
||||||
|
__tablename__ = "criterion_mappings"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
||||||
|
criterion_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||||
|
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
source_value: Mapped[str | None] = mapped_column(String(256))
|
||||||
|
encoding: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DiscoverySearchRecord(Base):
|
||||||
|
__tablename__ = "discovery_searches"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||||
|
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
schedule_interval_minutes: Mapped[int] = mapped_column(Integer, nullable=False, default=180)
|
||||||
|
broad_result_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=300)
|
||||||
|
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_successful_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class SearchRunRecord(Base):
|
||||||
|
__tablename__ = "search_runs"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
discovery_search_id: Mapped[UUID] = mapped_column(ForeignKey("discovery_searches.id"), nullable=False)
|
||||||
|
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
result_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
new_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
changed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
unchanged_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
missing_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
http_status_summary: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
|
||||||
|
error_code: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
|
||||||
|
|
||||||
|
class SourceProductReferenceRecord(Base):
|
||||||
|
__tablename__ = "source_product_references"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||||
|
source_external_id: Mapped[str | None] = mapped_column(String(256))
|
||||||
|
source_url: Mapped[str | None] = mapped_column(Text)
|
||||||
|
source_title: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||||
|
manufacturer: Mapped[str | None] = mapped_column(String(256))
|
||||||
|
model: Mapped[str | None] = mapped_column(String(256))
|
||||||
|
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
gtin_ean: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
normalized_spec_fingerprint: Mapped[str | None] = mapped_column(Text)
|
||||||
|
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
lifecycle_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
consecutive_missing_success_runs: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
|
||||||
|
|
||||||
|
|
||||||
|
class ProductObservationRecord(Base):
|
||||||
|
__tablename__ = "product_observations"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
||||||
|
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
|
||||||
|
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
raw_title: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||||
|
best_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||||
|
currency: Mapped[str | None] = mapped_column(String(3))
|
||||||
|
availability: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
seller_count: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
|
||||||
|
|
||||||
|
class ProductObservationAttributeRecord(Base):
|
||||||
|
__tablename__ = "product_observation_attributes"
|
||||||
|
__table_args__ = (UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
observation_id: Mapped[UUID] = mapped_column(ForeignKey("product_observations.id"), nullable=False)
|
||||||
|
criterion_definition_id: Mapped[UUID | None] = mapped_column(ForeignKey("criterion_definitions.id"))
|
||||||
|
attribute_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
raw_name: Mapped[str | None] = mapped_column(String(256))
|
||||||
|
raw_value: Mapped[Any] = mapped_column(JSONB)
|
||||||
|
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
||||||
|
unit: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
|
||||||
|
|
||||||
|
class SearchRunItemRecord(Base):
|
||||||
|
__tablename__ = "search_run_items"
|
||||||
|
__table_args__ = (UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
|
||||||
|
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
||||||
|
observation_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_observations.id"))
|
||||||
|
diff_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ProjectCandidateRecord(Base):
|
||||||
|
__tablename__ = "project_candidates"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
|
||||||
|
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
||||||
|
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
product_eligibility: Mapped[str | None] = mapped_column(String(16))
|
||||||
|
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class CandidateStatusHistoryRecord(Base):
|
||||||
|
__tablename__ = "candidate_status_history"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
candidate_id: Mapped[UUID] = mapped_column(ForeignKey("project_candidates.id"), nullable=False)
|
||||||
|
from_status: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
to_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
reason: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
105
backend/app/persistence/mapping.py
Normal file
105
backend/app/persistence/mapping.py
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
from app.domain.enums import (
|
||||||
|
Availability,
|
||||||
|
NormalizationStatus,
|
||||||
|
OfferCondition,
|
||||||
|
ValueType,
|
||||||
|
)
|
||||||
|
from app.domain.models import (
|
||||||
|
AttributeDefinition,
|
||||||
|
AttributeStatement,
|
||||||
|
Offer,
|
||||||
|
PriceObservation,
|
||||||
|
Product,
|
||||||
|
ProductVariant,
|
||||||
|
)
|
||||||
|
from app.persistence import models as orm
|
||||||
|
|
||||||
|
|
||||||
|
def decimal_or_none(value) -> Decimal | None:
|
||||||
|
return None if value is None else Decimal(str(value))
|
||||||
|
|
||||||
|
|
||||||
|
def to_definition(row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
|
||||||
|
return AttributeDefinition(
|
||||||
|
key=row.key,
|
||||||
|
value_type=ValueType(row.value_type),
|
||||||
|
canonical_unit=row.canonical_unit,
|
||||||
|
minimum=decimal_or_none(row.minimum),
|
||||||
|
maximum=decimal_or_none(row.maximum),
|
||||||
|
description=row.description,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def to_product(row: orm.ProductRecord) -> Product:
|
||||||
|
return Product(
|
||||||
|
id=row.id,
|
||||||
|
category=row.category,
|
||||||
|
brand=row.brand,
|
||||||
|
name=row.name,
|
||||||
|
model=row.model,
|
||||||
|
created_at=row.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def to_variant(row: orm.ProductVariantRecord) -> ProductVariant:
|
||||||
|
return ProductVariant(
|
||||||
|
id=row.id,
|
||||||
|
product_id=row.product_id,
|
||||||
|
name=row.name,
|
||||||
|
manufacturer_sku=row.manufacturer_sku,
|
||||||
|
identifiers=row.identifiers or {},
|
||||||
|
created_at=row.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def to_attribute(row: orm.VariantAttributeRecord) -> AttributeStatement:
|
||||||
|
return AttributeStatement(
|
||||||
|
id=row.id,
|
||||||
|
variant_id=row.variant_id,
|
||||||
|
attribute=row.attribute_key,
|
||||||
|
raw_value=row.raw_value,
|
||||||
|
normalized_value=row.normalized_value,
|
||||||
|
unit=row.unit,
|
||||||
|
normalization_status=NormalizationStatus(row.normalization_status),
|
||||||
|
normalizer_version=row.normalizer_version,
|
||||||
|
confidence=Decimal(str(row.confidence)),
|
||||||
|
source_id=row.source_id,
|
||||||
|
source_reference=row.source_reference,
|
||||||
|
observed_at=row.observed_at,
|
||||||
|
ingested_at=row.ingested_at,
|
||||||
|
normalization_error=row.normalization_error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def to_offer(row: orm.OfferRecord) -> Offer:
|
||||||
|
return Offer(
|
||||||
|
id=row.id,
|
||||||
|
variant_id=row.variant_id,
|
||||||
|
source_id=row.source_id,
|
||||||
|
external_id=row.external_id,
|
||||||
|
seller=row.seller,
|
||||||
|
url=row.url,
|
||||||
|
market=row.market,
|
||||||
|
currency=row.currency,
|
||||||
|
condition=OfferCondition(row.condition),
|
||||||
|
created_at=row.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def to_observation(row: orm.PriceObservationRecord) -> PriceObservation:
|
||||||
|
return PriceObservation(
|
||||||
|
id=row.id,
|
||||||
|
offer_id=row.offer_id,
|
||||||
|
source_id=row.source_id,
|
||||||
|
external_observation_id=row.external_observation_id,
|
||||||
|
observed_at=row.observed_at,
|
||||||
|
ingested_at=row.ingested_at,
|
||||||
|
item_price=Decimal(str(row.item_price)),
|
||||||
|
shipping_price=decimal_or_none(row.shipping_price),
|
||||||
|
currency=row.currency,
|
||||||
|
availability=Availability(row.availability),
|
||||||
|
raw_payload=row.raw_payload,
|
||||||
|
confidence=Decimal(str(row.confidence)),
|
||||||
|
)
|
||||||
249
backend/app/persistence/models.py
Normal file
249
backend/app/persistence/models.py
Normal file
|
|
@ -0,0 +1,249 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
CheckConstraint,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
ForeignKeyConstraint,
|
||||||
|
Numeric,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
)
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
||||||
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||||
|
|
||||||
|
|
||||||
|
class Base(DeclarativeBase):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class AttributeDefinitionRecord(Base):
|
||||||
|
__tablename__ = "attribute_definitions"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"value_type IN ('number', 'string', 'boolean', 'string_list')",
|
||||||
|
name="ck_attribute_definitions_value_type",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||||
|
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
canonical_unit: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
|
||||||
|
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
|
||||||
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRecord(Base):
|
||||||
|
__tablename__ = "sources"
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
adapter_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ScoringModelRecord(Base):
|
||||||
|
__tablename__ = "scoring_models"
|
||||||
|
|
||||||
|
version: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||||
|
algorithm_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ProductProfileRecord(Base):
|
||||||
|
__tablename__ = "product_profiles"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("version > 0", name="ck_product_profiles_version"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||||
|
version: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
category: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
|
||||||
|
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ProductRecord(Base):
|
||||||
|
__tablename__ = "products"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
category: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
brand: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
model: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
variants: Mapped[list["ProductVariantRecord"]] = relationship(back_populates="product")
|
||||||
|
|
||||||
|
|
||||||
|
class ProductVariantRecord(Base):
|
||||||
|
__tablename__ = "product_variants"
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
product_id: Mapped[UUID] = mapped_column(ForeignKey("products.id"), nullable=False, index=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
|
||||||
|
identifiers: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
product: Mapped[ProductRecord] = relationship(back_populates="variants")
|
||||||
|
|
||||||
|
|
||||||
|
class VariantAttributeRecord(Base):
|
||||||
|
__tablename__ = "variant_attributes"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
|
||||||
|
CheckConstraint(
|
||||||
|
"normalization_status IN ('normalized', 'unknown')",
|
||||||
|
name="ck_variant_attributes_status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False)
|
||||||
|
attribute_key: Mapped[str] = mapped_column(ForeignKey("attribute_definitions.key"), nullable=False)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
raw_value: Mapped[Any] = mapped_column(JSONB, nullable=False)
|
||||||
|
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
||||||
|
unit: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
normalization_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
normalizer_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
|
||||||
|
source_reference: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
normalization_error: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|
||||||
|
|
||||||
|
class OfferRecord(Base):
|
||||||
|
__tablename__ = "offers"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
|
||||||
|
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
|
||||||
|
CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
external_id: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
seller: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
url: Mapped[str | None] = mapped_column(Text)
|
||||||
|
market: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
||||||
|
condition: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class PriceObservationRecord(Base):
|
||||||
|
__tablename__ = "price_observations"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
|
||||||
|
CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
|
||||||
|
CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
|
||||||
|
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
|
||||||
|
CheckConstraint(
|
||||||
|
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
|
||||||
|
name="ck_price_obs_availability",
|
||||||
|
),
|
||||||
|
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
|
||||||
|
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||||
|
external_observation_id: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||||
|
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
item_price: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
|
||||||
|
shipping_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||||
|
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
||||||
|
availability: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
raw_payload: Mapped[Any] = mapped_column(JSONB, nullable=False)
|
||||||
|
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class EvaluationRecord(Base):
|
||||||
|
__tablename__ = "evaluations"
|
||||||
|
__table_args__ = (
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["profile_id", "profile_version"],
|
||||||
|
["product_profiles.id", "product_profiles.version"],
|
||||||
|
name="fk_evaluations_profile",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||||
|
name="ck_eval_product_eligibility",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||||
|
name="ck_eval_offer_eligibility",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||||
|
name="ck_eval_overall_eligibility",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
|
||||||
|
name="ck_eval_product_score",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
|
||||||
|
name="ck_eval_price_score",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
|
||||||
|
name="ck_eval_deal_score",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
|
||||||
|
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
|
||||||
|
profile_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
profile_version: Mapped[int] = mapped_column(nullable=False)
|
||||||
|
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
|
||||||
|
as_of: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
snapshot_cutoff: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
scoring_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
engine_build: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
product_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
offer_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
overall_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||||
|
price_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||||
|
deal_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||||
|
classification: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
price_reference_scope: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
input_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
result_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
components: Mapped[list["EvaluationComponentRecord"]] = relationship(back_populates="evaluation")
|
||||||
|
|
||||||
|
|
||||||
|
class EvaluationComponentRecord(Base):
|
||||||
|
__tablename__ = "evaluation_components"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||||
|
evaluation_id: Mapped[UUID] = mapped_column(ForeignKey("evaluations.id"), nullable=False)
|
||||||
|
kind: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
component_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
raw_score: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
||||||
|
weight: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
||||||
|
contribution: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
||||||
|
details: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
|
||||||
|
evaluation: Mapped[EvaluationRecord] = relationship(back_populates="components")
|
||||||
183
backend/app/persistence/repositories.py
Normal file
183
backend/app/persistence/repositories.py
Normal file
|
|
@ -0,0 +1,183 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
|
||||||
|
from app.domain.validation import parse_profile, parse_scoring_model
|
||||||
|
from app.persistence import models as orm
|
||||||
|
from app.persistence.mapping import to_attribute, to_definition, to_observation, to_offer, to_product, to_variant
|
||||||
|
|
||||||
|
|
||||||
|
class CatalogRepository:
|
||||||
|
def __init__(self, session: Session) -> None:
|
||||||
|
self.session = session
|
||||||
|
|
||||||
|
def get_definition(self, key: str) -> AttributeDefinition | None:
|
||||||
|
row = self.session.get(orm.AttributeDefinitionRecord, key)
|
||||||
|
return None if row is None else to_definition(row)
|
||||||
|
|
||||||
|
def list_definitions(self, limit: int, offset: int) -> tuple[list[AttributeDefinition], int]:
|
||||||
|
total = self.session.scalar(select(func.count()).select_from(orm.AttributeDefinitionRecord)) or 0
|
||||||
|
rows = self.session.scalars(
|
||||||
|
select(orm.AttributeDefinitionRecord).order_by(orm.AttributeDefinitionRecord.key).limit(limit).offset(offset)
|
||||||
|
).all()
|
||||||
|
return [to_definition(row) for row in rows], total
|
||||||
|
|
||||||
|
def add_definition(self, row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return to_definition(row)
|
||||||
|
|
||||||
|
def list_sources(self) -> list[orm.SourceRecord]:
|
||||||
|
return list(self.session.scalars(select(orm.SourceRecord).order_by(orm.SourceRecord.id)).all())
|
||||||
|
|
||||||
|
def get_source(self, source_id: str) -> orm.SourceRecord | None:
|
||||||
|
return self.session.get(orm.SourceRecord, source_id)
|
||||||
|
|
||||||
|
def get_scoring_model(self, version: str):
|
||||||
|
row = self.session.get(orm.ScoringModelRecord, version)
|
||||||
|
return None if row is None else parse_scoring_model(row.definition)
|
||||||
|
|
||||||
|
def scoring_versions(self) -> set[str]:
|
||||||
|
return set(self.session.scalars(select(orm.ScoringModelRecord.version)).all())
|
||||||
|
|
||||||
|
def all_definitions(self) -> dict[str, AttributeDefinition]:
|
||||||
|
rows = self.session.scalars(select(orm.AttributeDefinitionRecord)).all()
|
||||||
|
return {row.key: to_definition(row) for row in rows}
|
||||||
|
|
||||||
|
def get_profile(self, profile_id: str, version: int):
|
||||||
|
row = self.session.get(orm.ProductProfileRecord, (profile_id, version))
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
model = self.get_scoring_model(row.scoring_version)
|
||||||
|
if model is None:
|
||||||
|
return None
|
||||||
|
return parse_profile(
|
||||||
|
row.definition,
|
||||||
|
attribute_definitions=self.all_definitions(),
|
||||||
|
scoring_versions=self.scoring_versions(),
|
||||||
|
supported_strategies=set(model.supported_preference_strategies),
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_profile(self, row: orm.ProductProfileRecord):
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return row
|
||||||
|
|
||||||
|
def list_profiles(self, limit: int, offset: int) -> tuple[list[orm.ProductProfileRecord], int]:
|
||||||
|
total = self.session.scalar(select(func.count()).select_from(orm.ProductProfileRecord)) or 0
|
||||||
|
rows = self.session.scalars(
|
||||||
|
select(orm.ProductProfileRecord)
|
||||||
|
.order_by(orm.ProductProfileRecord.id, orm.ProductProfileRecord.version)
|
||||||
|
.limit(limit)
|
||||||
|
.offset(offset)
|
||||||
|
).all()
|
||||||
|
return list(rows), total
|
||||||
|
|
||||||
|
def get_product(self, product_id: UUID) -> Product | None:
|
||||||
|
row = self.session.get(orm.ProductRecord, product_id)
|
||||||
|
return None if row is None else to_product(row)
|
||||||
|
|
||||||
|
def add_product(self, row: orm.ProductRecord) -> Product:
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return to_product(row)
|
||||||
|
|
||||||
|
def get_variant(self, variant_id: UUID) -> ProductVariant | None:
|
||||||
|
row = self.session.get(orm.ProductVariantRecord, variant_id)
|
||||||
|
return None if row is None else to_variant(row)
|
||||||
|
|
||||||
|
def add_variant(self, row: orm.ProductVariantRecord) -> ProductVariant:
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return to_variant(row)
|
||||||
|
|
||||||
|
def list_attributes(self, variant_id: UUID) -> list[AttributeStatement]:
|
||||||
|
rows = self.session.scalars(
|
||||||
|
select(orm.VariantAttributeRecord)
|
||||||
|
.where(orm.VariantAttributeRecord.variant_id == variant_id)
|
||||||
|
.order_by(
|
||||||
|
orm.VariantAttributeRecord.observed_at,
|
||||||
|
orm.VariantAttributeRecord.ingested_at,
|
||||||
|
orm.VariantAttributeRecord.id,
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return [to_attribute(row) for row in rows]
|
||||||
|
|
||||||
|
def get_attribute(self, attribute_id: UUID) -> AttributeStatement | None:
|
||||||
|
row = self.session.get(orm.VariantAttributeRecord, attribute_id)
|
||||||
|
return None if row is None else to_attribute(row)
|
||||||
|
|
||||||
|
def add_attribute(self, row: orm.VariantAttributeRecord) -> AttributeStatement:
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return to_attribute(row)
|
||||||
|
|
||||||
|
def get_offer(self, offer_id: UUID) -> Offer | None:
|
||||||
|
row = self.session.get(orm.OfferRecord, offer_id)
|
||||||
|
return None if row is None else to_offer(row)
|
||||||
|
|
||||||
|
def get_offer_by_external(self, source_id: str, external_id: str) -> Offer | None:
|
||||||
|
row = self.session.scalar(
|
||||||
|
select(orm.OfferRecord).where(
|
||||||
|
orm.OfferRecord.source_id == source_id,
|
||||||
|
orm.OfferRecord.external_id == external_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None if row is None else to_offer(row)
|
||||||
|
|
||||||
|
def add_offer(self, row: orm.OfferRecord) -> Offer:
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return to_offer(row)
|
||||||
|
|
||||||
|
def list_observations(self, offer_id: UUID, limit: int | None = None, offset: int = 0) -> tuple[list[PriceObservation], int]:
|
||||||
|
total = (
|
||||||
|
self.session.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(orm.PriceObservationRecord)
|
||||||
|
.where(orm.PriceObservationRecord.offer_id == offer_id)
|
||||||
|
)
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
query = (
|
||||||
|
select(orm.PriceObservationRecord)
|
||||||
|
.where(orm.PriceObservationRecord.offer_id == offer_id)
|
||||||
|
.order_by(
|
||||||
|
orm.PriceObservationRecord.observed_at,
|
||||||
|
orm.PriceObservationRecord.ingested_at,
|
||||||
|
orm.PriceObservationRecord.id,
|
||||||
|
)
|
||||||
|
.offset(offset)
|
||||||
|
)
|
||||||
|
if limit is not None:
|
||||||
|
query = query.limit(limit)
|
||||||
|
rows = self.session.scalars(query).all()
|
||||||
|
return [to_observation(row) for row in rows], total
|
||||||
|
|
||||||
|
def get_observation_by_external(self, offer_id: UUID, external_id: str) -> orm.PriceObservationRecord | None:
|
||||||
|
return self.session.scalar(
|
||||||
|
select(orm.PriceObservationRecord).where(
|
||||||
|
orm.PriceObservationRecord.offer_id == offer_id,
|
||||||
|
orm.PriceObservationRecord.external_observation_id == external_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_observation(self, row: orm.PriceObservationRecord) -> PriceObservation:
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return to_observation(row)
|
||||||
|
|
||||||
|
def get_evaluation(self, evaluation_id: UUID) -> orm.EvaluationRecord | None:
|
||||||
|
return self.session.get(orm.EvaluationRecord, evaluation_id)
|
||||||
|
|
||||||
|
def add_evaluation(self, row: orm.EvaluationRecord) -> orm.EvaluationRecord:
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
return row
|
||||||
|
|
||||||
|
def add_component(self, row: orm.EvaluationComponentRecord) -> None:
|
||||||
|
self.session.add(row)
|
||||||
0
backend/app/scoring/__init__.py
Normal file
0
backend/app/scoring/__init__.py
Normal file
42
backend/app/scoring/deal.py
Normal file
42
backend/app/scoring/deal.py
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
from app.domain.enums import Classification, Eligibility
|
||||||
|
from app.domain.models import ScoringModel
|
||||||
|
from app.scoring.mathutil import clamp
|
||||||
|
|
||||||
|
|
||||||
|
def classify_deal_score(score: Decimal, model: ScoringModel) -> str:
|
||||||
|
for bound in model.classification_upper_bounds:
|
||||||
|
if bound.inclusive and score <= bound.upper:
|
||||||
|
return bound.label
|
||||||
|
if not bound.inclusive and score < bound.upper:
|
||||||
|
return bound.label
|
||||||
|
return Classification.EXCEPTIONAL.value
|
||||||
|
|
||||||
|
|
||||||
|
def score_deal(product_score: Decimal, price_score: Decimal, model: ScoringModel) -> Decimal:
|
||||||
|
return clamp(model.product_weight * product_score + model.price_weight * price_score)
|
||||||
|
|
||||||
|
|
||||||
|
def eligibility_from_parts(product: Eligibility, offer: Eligibility) -> Eligibility:
|
||||||
|
if Eligibility.INELIGIBLE in {product, offer}:
|
||||||
|
return Eligibility.INELIGIBLE
|
||||||
|
if Eligibility.UNKNOWN in {product, offer}:
|
||||||
|
return Eligibility.UNKNOWN
|
||||||
|
return Eligibility.ELIGIBLE
|
||||||
|
|
||||||
|
|
||||||
|
def derive_eligible(overall: Eligibility) -> bool | None:
|
||||||
|
if overall == Eligibility.ELIGIBLE:
|
||||||
|
return True
|
||||||
|
if overall == Eligibility.INELIGIBLE:
|
||||||
|
return False
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def classification_for(overall: Eligibility, deal_score: Decimal | None, model: ScoringModel) -> str:
|
||||||
|
if overall == Eligibility.INELIGIBLE:
|
||||||
|
return Classification.INELIGIBLE.value
|
||||||
|
if overall == Eligibility.UNKNOWN or deal_score is None:
|
||||||
|
return Classification.INSUFFICIENT_DATA.value
|
||||||
|
return classify_deal_score(deal_score, model)
|
||||||
545
backend/app/scoring/engine.py
Normal file
545
backend/app/scoring/engine.py
Normal file
|
|
@ -0,0 +1,545 @@
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.domain.attributes import select_attribute_snapshot
|
||||||
|
from app.domain.constraints import evaluate_constraint
|
||||||
|
from app.domain.enums import (
|
||||||
|
Availability,
|
||||||
|
ConstraintStatus,
|
||||||
|
Eligibility,
|
||||||
|
PriceReferenceScope,
|
||||||
|
)
|
||||||
|
from app.domain.models import (
|
||||||
|
AttributeDefinition,
|
||||||
|
AttributeStatement,
|
||||||
|
ConstraintResult,
|
||||||
|
EvaluationResult,
|
||||||
|
GateResult,
|
||||||
|
Offer,
|
||||||
|
PriceObservation,
|
||||||
|
Product,
|
||||||
|
ProductVariant,
|
||||||
|
Profile,
|
||||||
|
Reason,
|
||||||
|
ScoreComponent,
|
||||||
|
ScoringModel,
|
||||||
|
)
|
||||||
|
from app.domain.serialize import jsonable
|
||||||
|
from app.scoring.deal import classification_for, derive_eligible, eligibility_from_parts, score_deal
|
||||||
|
from app.scoring.mathutil import round_score
|
||||||
|
from app.scoring.price import (
|
||||||
|
build_offer_reference_history,
|
||||||
|
price_statistics_from_windows,
|
||||||
|
score_price,
|
||||||
|
select_current_observation,
|
||||||
|
)
|
||||||
|
from app.scoring.product import score_product
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_deal(
|
||||||
|
*,
|
||||||
|
profile: Profile,
|
||||||
|
scoring_model: ScoringModel,
|
||||||
|
product: Product,
|
||||||
|
variant: ProductVariant,
|
||||||
|
offer: Offer,
|
||||||
|
attributes: tuple[AttributeStatement, ...],
|
||||||
|
observations: tuple[PriceObservation, ...],
|
||||||
|
definitions: dict[str, AttributeDefinition],
|
||||||
|
as_of: datetime,
|
||||||
|
snapshot_cutoff: datetime,
|
||||||
|
engine_build: str,
|
||||||
|
) -> EvaluationResult:
|
||||||
|
observations = tuple(item for item in observations if item.offer_id == offer.id)
|
||||||
|
snapshot = select_attribute_snapshot(attributes, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
|
||||||
|
constraint_results = tuple(
|
||||||
|
evaluate_constraint(
|
||||||
|
constraint,
|
||||||
|
snapshot.get(constraint.attribute),
|
||||||
|
definitions[constraint.attribute],
|
||||||
|
scoring_model.minimum_confidence,
|
||||||
|
)
|
||||||
|
for constraint in profile.constraints
|
||||||
|
)
|
||||||
|
product_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
|
||||||
|
product_score_raw, coverage, preference_components = score_product(
|
||||||
|
profile.preferences,
|
||||||
|
snapshot,
|
||||||
|
constraint_results,
|
||||||
|
scoring_model,
|
||||||
|
)
|
||||||
|
|
||||||
|
current = select_current_observation(observations, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
|
||||||
|
offer_gates, offer_eligibility = _offer_eligibility(profile, offer, current, scoring_model, as_of)
|
||||||
|
overall = eligibility_from_parts(product_eligibility, offer_eligibility)
|
||||||
|
|
||||||
|
windows: dict[int, Any] = {}
|
||||||
|
price_score_raw: Decimal | None = None
|
||||||
|
budget_score: Decimal | None = None
|
||||||
|
history_score: Decimal | None = None
|
||||||
|
price_basis = None
|
||||||
|
history_warning: str | None = None
|
||||||
|
if _can_score_price(current, profile, scoring_model, as_of):
|
||||||
|
windows = build_offer_reference_history(observations, current, scoring_model)
|
||||||
|
price_score_raw, budget_score, history_score, price_basis, history_warning = score_price(
|
||||||
|
current_price=current.total_price,
|
||||||
|
reference_history=windows[90].values,
|
||||||
|
budget=profile.budget,
|
||||||
|
model=scoring_model,
|
||||||
|
)
|
||||||
|
|
||||||
|
stats = price_statistics_from_windows(current, windows)
|
||||||
|
deal_score_raw = None
|
||||||
|
if overall == Eligibility.ELIGIBLE and price_score_raw is not None:
|
||||||
|
deal_score_raw = score_deal(product_score_raw, price_score_raw, scoring_model)
|
||||||
|
|
||||||
|
classification = classification_for(overall, deal_score_raw, scoring_model)
|
||||||
|
reasons, warnings = _reasons(
|
||||||
|
constraint_results,
|
||||||
|
product_gates,
|
||||||
|
offer_gates,
|
||||||
|
stats,
|
||||||
|
current,
|
||||||
|
history_warning,
|
||||||
|
price_score_raw,
|
||||||
|
)
|
||||||
|
components = _all_components(
|
||||||
|
preference_components,
|
||||||
|
product_gates,
|
||||||
|
offer_gates,
|
||||||
|
budget_score,
|
||||||
|
history_score,
|
||||||
|
scoring_model,
|
||||||
|
)
|
||||||
|
input_snapshot = {
|
||||||
|
"profile": profile.definition,
|
||||||
|
"scoring_model": scoring_model.definition,
|
||||||
|
"product": jsonable(_product_dict(product)),
|
||||||
|
"variant": jsonable(_variant_dict(variant)),
|
||||||
|
"offer": jsonable(_offer_dict(offer)),
|
||||||
|
"attribute_snapshot": {
|
||||||
|
key: jsonable(_attribute_dict(value))
|
||||||
|
for key, value in snapshot.items()
|
||||||
|
},
|
||||||
|
"current_observation": None if current is None else jsonable(_observation_dict(current)),
|
||||||
|
"history_observation_ids": [str(item) for item in stats.used_observation_ids],
|
||||||
|
"as_of": jsonable(as_of),
|
||||||
|
"snapshot_cutoff": jsonable(snapshot_cutoff),
|
||||||
|
"price_reference_scope": PriceReferenceScope.OFFER.value,
|
||||||
|
"unrounded": {
|
||||||
|
"product_score": jsonable(product_score_raw),
|
||||||
|
"price_score": jsonable(price_score_raw),
|
||||||
|
"deal_score": jsonable(deal_score_raw),
|
||||||
|
"budget_score": jsonable(budget_score),
|
||||||
|
"history_score": jsonable(history_score),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return EvaluationResult(
|
||||||
|
profile_id=profile.id,
|
||||||
|
profile_version=profile.version,
|
||||||
|
scoring_version=scoring_model.version,
|
||||||
|
scoring_hash=scoring_model.definition_hash,
|
||||||
|
engine_build=engine_build,
|
||||||
|
variant_id=variant.id,
|
||||||
|
offer_id=offer.id,
|
||||||
|
as_of=as_of,
|
||||||
|
snapshot_cutoff=snapshot_cutoff,
|
||||||
|
product_eligibility=product_eligibility,
|
||||||
|
offer_eligibility=offer_eligibility,
|
||||||
|
overall_eligibility=overall,
|
||||||
|
eligible=derive_eligible(overall),
|
||||||
|
product_score=round_score(product_score_raw),
|
||||||
|
product_score_raw=product_score_raw,
|
||||||
|
price_score=None if price_score_raw is None else round_score(price_score_raw),
|
||||||
|
price_score_raw=price_score_raw,
|
||||||
|
deal_score=None if deal_score_raw is None else round_score(deal_score_raw),
|
||||||
|
deal_score_raw=deal_score_raw,
|
||||||
|
classification=classification,
|
||||||
|
price_basis=price_basis,
|
||||||
|
product_coverage=coverage,
|
||||||
|
constraint_results=constraint_results,
|
||||||
|
product_gates=product_gates,
|
||||||
|
offer_gates=offer_gates,
|
||||||
|
components=components,
|
||||||
|
price_statistics=stats,
|
||||||
|
price_reference_scope=PriceReferenceScope.OFFER,
|
||||||
|
reasons=reasons,
|
||||||
|
warnings=warnings,
|
||||||
|
input_snapshot=input_snapshot,
|
||||||
|
failed_constraints=tuple(
|
||||||
|
item for item in constraint_results if item.status == ConstraintStatus.FAIL.value
|
||||||
|
),
|
||||||
|
unknown_constraints=tuple(
|
||||||
|
item for item in constraint_results if item.status == ConstraintStatus.UNKNOWN.value
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def statement_price_confidence(observation: PriceObservation, model: ScoringModel) -> bool:
|
||||||
|
return observation.confidence >= model.minimum_confidence
|
||||||
|
|
||||||
|
|
||||||
|
def _fresh_enough(observation: PriceObservation, as_of: datetime, model: ScoringModel) -> bool:
|
||||||
|
return as_of - observation.observed_at <= timedelta(hours=model.maximum_price_age_hours)
|
||||||
|
|
||||||
|
|
||||||
|
def _product_eligibility(
|
||||||
|
profile: Profile,
|
||||||
|
product: Product,
|
||||||
|
constraint_results: tuple[ConstraintResult, ...],
|
||||||
|
) -> tuple[tuple[GateResult, ...], Eligibility]:
|
||||||
|
category_ok = product.category == profile.category
|
||||||
|
category_gate = GateResult(
|
||||||
|
key="category",
|
||||||
|
status="PASS" if category_ok else "FAIL",
|
||||||
|
reason_code="CATEGORY_MATCH" if category_ok else "CATEGORY_MISMATCH",
|
||||||
|
details={"expected": profile.category, "actual": product.category},
|
||||||
|
)
|
||||||
|
mandatory = [item for item in constraint_results if item.mandatory]
|
||||||
|
if not category_ok or any(item.status == ConstraintStatus.FAIL.value for item in mandatory):
|
||||||
|
eligibility = Eligibility.INELIGIBLE
|
||||||
|
elif any(item.status == ConstraintStatus.UNKNOWN.value for item in mandatory):
|
||||||
|
eligibility = Eligibility.UNKNOWN
|
||||||
|
else:
|
||||||
|
eligibility = Eligibility.ELIGIBLE
|
||||||
|
return (category_gate,), eligibility
|
||||||
|
|
||||||
|
|
||||||
|
def _offer_eligibility(
|
||||||
|
profile: Profile,
|
||||||
|
offer: Offer,
|
||||||
|
current: PriceObservation | None,
|
||||||
|
model: ScoringModel,
|
||||||
|
as_of: datetime,
|
||||||
|
) -> tuple[tuple[GateResult, ...], Eligibility]:
|
||||||
|
gates: list[GateResult] = []
|
||||||
|
|
||||||
|
market_ok = offer.market == profile.market
|
||||||
|
gates.append(_gate("market", market_ok, "MARKET_MATCH", "MARKET_MISMATCH", offer.market, profile.market))
|
||||||
|
|
||||||
|
condition_ok = offer.condition in profile.allowed_conditions
|
||||||
|
gates.append(
|
||||||
|
_gate(
|
||||||
|
"condition",
|
||||||
|
condition_ok,
|
||||||
|
"CONDITION_ALLOWED",
|
||||||
|
"CONDITION_NOT_ALLOWED",
|
||||||
|
offer.condition.value,
|
||||||
|
[item.value for item in profile.allowed_conditions],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
currency_ok = offer.currency == profile.currency
|
||||||
|
gates.append(_gate("currency", currency_ok, "CURRENCY_MATCH", "CURRENCY_MISMATCH", offer.currency, profile.currency))
|
||||||
|
|
||||||
|
if current is None:
|
||||||
|
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_MISSING", {}))
|
||||||
|
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_MISSING", {}))
|
||||||
|
gates.append(GateResult("price_confidence", "UNKNOWN", "PRICE_MISSING", {}))
|
||||||
|
gates.append(GateResult("shipping", "UNKNOWN", "SHIPPING_UNKNOWN", {}))
|
||||||
|
gates.append(GateResult("budget", "UNKNOWN", "PRICE_MISSING", {}))
|
||||||
|
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
|
||||||
|
return tuple(gates), _combine_offer_gates(gates)
|
||||||
|
|
||||||
|
fresh = _fresh_enough(current, as_of, model)
|
||||||
|
if not fresh:
|
||||||
|
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_STALE", {"observed_at": current.observed_at.isoformat()}))
|
||||||
|
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_STALE", {}))
|
||||||
|
else:
|
||||||
|
gates.append(GateResult("price_present", "PASS", "PRICE_PRESENT", {"observation_id": str(current.id)}))
|
||||||
|
gates.append(GateResult("price_fresh", "PASS", "PRICE_FRESH", {}))
|
||||||
|
|
||||||
|
confident = statement_price_confidence(current, model)
|
||||||
|
gates.append(
|
||||||
|
GateResult(
|
||||||
|
"price_confidence",
|
||||||
|
"PASS" if confident else "UNKNOWN",
|
||||||
|
"PRICE_CONFIDENCE_OK" if confident else "PRICE_LOW_CONFIDENCE",
|
||||||
|
{"confidence": str(current.confidence)},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
shipping_known = current.shipping_price is not None
|
||||||
|
gates.append(
|
||||||
|
GateResult(
|
||||||
|
"shipping",
|
||||||
|
"PASS" if shipping_known else "UNKNOWN",
|
||||||
|
"SHIPPING_KNOWN" if shipping_known else "SHIPPING_UNKNOWN",
|
||||||
|
{},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
total = current.total_price
|
||||||
|
if total is None or not currency_ok:
|
||||||
|
gates.append(GateResult("budget", "UNKNOWN", "PRICE_INCOMPLETE", {}))
|
||||||
|
elif total <= profile.budget.maximum:
|
||||||
|
gates.append(GateResult("budget", "PASS", "WITHIN_BUDGET", {"total": str(total)}))
|
||||||
|
else:
|
||||||
|
gates.append(GateResult("budget", "FAIL", "OVER_BUDGET", {"total": str(total)}))
|
||||||
|
|
||||||
|
if current.availability == Availability.IN_STOCK:
|
||||||
|
gates.append(GateResult("availability", "PASS", "IN_STOCK", {}))
|
||||||
|
elif current.availability == Availability.OUT_OF_STOCK:
|
||||||
|
gates.append(GateResult("availability", "FAIL", "OUT_OF_STOCK", {}))
|
||||||
|
else:
|
||||||
|
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
|
||||||
|
|
||||||
|
if current.currency != offer.currency:
|
||||||
|
gates.append(GateResult("observation_currency", "FAIL", "CURRENCY_MISMATCH", {}))
|
||||||
|
|
||||||
|
if current.source_id != offer.source_id:
|
||||||
|
gates.append(GateResult("observation_source", "FAIL", "SOURCE_MISMATCH", {}))
|
||||||
|
|
||||||
|
return tuple(gates), _combine_offer_gates(gates)
|
||||||
|
|
||||||
|
|
||||||
|
def _gate(key: str, ok: bool, pass_code: str, fail_code: str, actual: Any, expected: Any) -> GateResult:
|
||||||
|
return GateResult(
|
||||||
|
key=key,
|
||||||
|
status="PASS" if ok else "FAIL",
|
||||||
|
reason_code=pass_code if ok else fail_code,
|
||||||
|
details={"actual": actual, "expected": expected},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _can_score_price(
|
||||||
|
current: PriceObservation | None,
|
||||||
|
profile: Profile,
|
||||||
|
model: ScoringModel,
|
||||||
|
as_of: datetime,
|
||||||
|
) -> bool:
|
||||||
|
if current is None:
|
||||||
|
return False
|
||||||
|
if current.currency != profile.currency:
|
||||||
|
return False
|
||||||
|
if current.total_price is None:
|
||||||
|
return False
|
||||||
|
if not _fresh_enough(current, as_of, model):
|
||||||
|
return False
|
||||||
|
return statement_price_confidence(current, model)
|
||||||
|
|
||||||
|
|
||||||
|
def _combine_offer_gates(gates: Sequence[GateResult]) -> Eligibility:
|
||||||
|
statuses = [gate.status for gate in gates]
|
||||||
|
if "FAIL" in statuses:
|
||||||
|
return Eligibility.INELIGIBLE
|
||||||
|
if "UNKNOWN" in statuses:
|
||||||
|
return Eligibility.UNKNOWN
|
||||||
|
return Eligibility.ELIGIBLE
|
||||||
|
|
||||||
|
|
||||||
|
def _reasons(
|
||||||
|
constraints: tuple[ConstraintResult, ...],
|
||||||
|
product_gates: tuple[GateResult, ...],
|
||||||
|
offer_gates: tuple[GateResult, ...],
|
||||||
|
stats,
|
||||||
|
current: PriceObservation | None,
|
||||||
|
history_warning: str | None,
|
||||||
|
price_score_raw: Decimal | None,
|
||||||
|
) -> tuple[tuple[Reason, ...], tuple[str, ...]]:
|
||||||
|
reasons: list[Reason] = []
|
||||||
|
warnings: list[str] = []
|
||||||
|
for gate in (*product_gates, *offer_gates):
|
||||||
|
if gate.status != "PASS":
|
||||||
|
reasons.append(Reason(gate.reason_code, _gate_message(gate)))
|
||||||
|
for item in constraints:
|
||||||
|
if item.status != ConstraintStatus.PASS.value:
|
||||||
|
reasons.append(Reason(item.reason_code, _constraint_message(item)))
|
||||||
|
if history_warning:
|
||||||
|
warnings.append(history_warning)
|
||||||
|
reasons.append(Reason(history_warning, "Zu wenige Historientage für den 90-Tage-Median."))
|
||||||
|
if stats.lowest_365d is not None and not stats.lowest_365d_is_full_year:
|
||||||
|
warnings.append("LOWEST_365D_PARTIAL")
|
||||||
|
reasons.append(
|
||||||
|
Reason(
|
||||||
|
"LOWEST_365D_PARTIAL",
|
||||||
|
f"lowest_365d {stats.lowest_365d} ist das Minimum der vorhandenen {stats.lowest_365d_days} Tage, kein belegtes Jahrestief.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
price_score_raw is not None
|
||||||
|
and current is not None
|
||||||
|
and current.total_price is not None
|
||||||
|
and stats.median_90d is not None
|
||||||
|
and stats.median_90d != 0
|
||||||
|
):
|
||||||
|
delta = (stats.median_90d - current.total_price) / stats.median_90d * Decimal("100")
|
||||||
|
percent = round_score(delta)
|
||||||
|
if current.total_price < stats.median_90d:
|
||||||
|
reasons.append(
|
||||||
|
Reason(
|
||||||
|
"PRICE_BELOW_MEDIAN",
|
||||||
|
f"Gesamtpreis {_de_decimal(percent)} % unter dem Median der beobachteten 90-Tage-Historie.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif current.total_price > stats.median_90d:
|
||||||
|
reasons.append(
|
||||||
|
Reason(
|
||||||
|
"PRICE_ABOVE_MEDIAN",
|
||||||
|
f"Gesamtpreis {_de_decimal(percent.copy_abs())} % über dem Median der beobachteten 90-Tage-Historie.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
reasons.append(Reason("PRICE_AT_MEDIAN", "Gesamtpreis entspricht dem Median der 90-Tage-Historie."))
|
||||||
|
return tuple(reasons), tuple(warnings)
|
||||||
|
|
||||||
|
|
||||||
|
def _de_decimal(value: Decimal) -> str:
|
||||||
|
return format(value, "f").replace(".", ",")
|
||||||
|
|
||||||
|
|
||||||
|
def _gate_message(gate: GateResult) -> str:
|
||||||
|
messages = {
|
||||||
|
"CATEGORY_MISMATCH": "Die Produktkategorie entspricht nicht dem Profil.",
|
||||||
|
"MARKET_MISMATCH": "Der Angebotsmarkt entspricht nicht dem Profil.",
|
||||||
|
"CONDITION_NOT_ALLOWED": "Der Angebotszustand ist im Profil nicht erlaubt.",
|
||||||
|
"CURRENCY_MISMATCH": "Die Währung entspricht nicht dem Profil.",
|
||||||
|
"PRICE_MISSING": "Es liegt keine aktuelle Preisbeobachtung vor.",
|
||||||
|
"PRICE_STALE": "Die aktuelle Preisbeobachtung ist nicht frisch genug.",
|
||||||
|
"PRICE_LOW_CONFIDENCE": "Die Preis-Confidence liegt unter der Mindestschwelle.",
|
||||||
|
"SHIPPING_UNKNOWN": "Obligatorische Versandkosten sind unbekannt.",
|
||||||
|
"OVER_BUDGET": "Der Gesamtpreis liegt über dem Budgetmaximum.",
|
||||||
|
"OUT_OF_STOCK": "Das Angebot ist nicht lieferbar.",
|
||||||
|
"AVAILABILITY_UNKNOWN": "Die Verfügbarkeit ist unbekannt.",
|
||||||
|
"SOURCE_MISMATCH": "Die Beobachtungsquelle weicht vom Angebot ab.",
|
||||||
|
}
|
||||||
|
return messages.get(gate.reason_code, gate.reason_code)
|
||||||
|
|
||||||
|
|
||||||
|
def _constraint_message(item: ConstraintResult) -> str:
|
||||||
|
if item.status == ConstraintStatus.FAIL.value:
|
||||||
|
return f"Constraint {item.constraint_id} ist nicht erfüllt."
|
||||||
|
return f"Constraint {item.constraint_id} kann nicht bewertet werden."
|
||||||
|
|
||||||
|
|
||||||
|
def _all_components(
|
||||||
|
preference_components: tuple[ScoreComponent, ...],
|
||||||
|
product_gates: tuple[GateResult, ...],
|
||||||
|
offer_gates: tuple[GateResult, ...],
|
||||||
|
budget_score: Decimal | None,
|
||||||
|
history_score: Decimal | None,
|
||||||
|
model: ScoringModel,
|
||||||
|
) -> tuple[ScoreComponent, ...]:
|
||||||
|
extra: list[ScoreComponent] = list(preference_components)
|
||||||
|
if budget_score is not None:
|
||||||
|
extra.append(
|
||||||
|
ScoreComponent(
|
||||||
|
kind="budget",
|
||||||
|
component_key="budget",
|
||||||
|
raw_score=budget_score,
|
||||||
|
weight=model.budget_weight,
|
||||||
|
contribution=model.budget_weight * budget_score,
|
||||||
|
details={},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if history_score is not None:
|
||||||
|
extra.append(
|
||||||
|
ScoreComponent(
|
||||||
|
kind="history",
|
||||||
|
component_key="history",
|
||||||
|
raw_score=history_score,
|
||||||
|
weight=model.history_weight,
|
||||||
|
contribution=model.history_weight * history_score,
|
||||||
|
details={},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for gate in product_gates:
|
||||||
|
extra.append(
|
||||||
|
ScoreComponent(
|
||||||
|
kind="product_gate",
|
||||||
|
component_key=gate.key,
|
||||||
|
raw_score=None,
|
||||||
|
weight=None,
|
||||||
|
contribution=None,
|
||||||
|
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for gate in offer_gates:
|
||||||
|
extra.append(
|
||||||
|
ScoreComponent(
|
||||||
|
kind="offer_gate",
|
||||||
|
component_key=gate.key,
|
||||||
|
raw_score=None,
|
||||||
|
weight=None,
|
||||||
|
contribution=None,
|
||||||
|
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(extra)
|
||||||
|
|
||||||
|
|
||||||
|
def _product_dict(product: Product) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": product.id,
|
||||||
|
"category": product.category,
|
||||||
|
"brand": product.brand,
|
||||||
|
"name": product.name,
|
||||||
|
"model": product.model,
|
||||||
|
"created_at": product.created_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _variant_dict(variant: ProductVariant) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": variant.id,
|
||||||
|
"product_id": variant.product_id,
|
||||||
|
"name": variant.name,
|
||||||
|
"manufacturer_sku": variant.manufacturer_sku,
|
||||||
|
"identifiers": variant.identifiers,
|
||||||
|
"created_at": variant.created_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _offer_dict(offer: Offer) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": offer.id,
|
||||||
|
"variant_id": offer.variant_id,
|
||||||
|
"source_id": offer.source_id,
|
||||||
|
"external_id": offer.external_id,
|
||||||
|
"seller": offer.seller,
|
||||||
|
"url": offer.url,
|
||||||
|
"market": offer.market,
|
||||||
|
"currency": offer.currency,
|
||||||
|
"condition": offer.condition,
|
||||||
|
"created_at": offer.created_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _attribute_dict(statement: AttributeStatement) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": statement.id,
|
||||||
|
"variant_id": statement.variant_id,
|
||||||
|
"attribute": statement.attribute,
|
||||||
|
"raw_value": statement.raw_value,
|
||||||
|
"normalized_value": statement.normalized_value,
|
||||||
|
"unit": statement.unit,
|
||||||
|
"normalization_status": statement.normalization_status,
|
||||||
|
"normalizer_version": statement.normalizer_version,
|
||||||
|
"confidence": statement.confidence,
|
||||||
|
"source_id": statement.source_id,
|
||||||
|
"source_reference": statement.source_reference,
|
||||||
|
"observed_at": statement.observed_at,
|
||||||
|
"ingested_at": statement.ingested_at,
|
||||||
|
"normalization_error": statement.normalization_error,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _observation_dict(observation: PriceObservation) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": observation.id,
|
||||||
|
"offer_id": observation.offer_id,
|
||||||
|
"source_id": observation.source_id,
|
||||||
|
"external_observation_id": observation.external_observation_id,
|
||||||
|
"observed_at": observation.observed_at,
|
||||||
|
"ingested_at": observation.ingested_at,
|
||||||
|
"item_price": observation.item_price,
|
||||||
|
"shipping_price": observation.shipping_price,
|
||||||
|
"currency": observation.currency,
|
||||||
|
"availability": observation.availability,
|
||||||
|
"raw_payload": observation.raw_payload,
|
||||||
|
"confidence": observation.confidence,
|
||||||
|
"total_price": observation.total_price,
|
||||||
|
}
|
||||||
|
|
||||||
21
backend/app/scoring/mathutil.py
Normal file
21
backend/app/scoring/mathutil.py
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
from decimal import ROUND_HALF_UP, Decimal
|
||||||
|
|
||||||
|
HUNDRED = Decimal("100")
|
||||||
|
ZERO = Decimal("0")
|
||||||
|
TWO_PLACES = Decimal("0.01")
|
||||||
|
|
||||||
|
|
||||||
|
def clamp(value: Decimal) -> Decimal:
|
||||||
|
return min(HUNDRED, max(ZERO, value))
|
||||||
|
|
||||||
|
|
||||||
|
def round_score(value: Decimal) -> Decimal:
|
||||||
|
return value.quantize(TWO_PLACES, rounding=ROUND_HALF_UP)
|
||||||
|
|
||||||
|
|
||||||
|
def as_decimal(value: object) -> Decimal:
|
||||||
|
if isinstance(value, Decimal):
|
||||||
|
return value
|
||||||
|
if isinstance(value, bool) or value is None:
|
||||||
|
raise TypeError("value is not numeric")
|
||||||
|
return Decimal(str(value))
|
||||||
148
backend/app/scoring/price.py
Normal file
148
backend/app/scoring/price.py
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import date, datetime, timedelta
|
||||||
|
from decimal import Decimal
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.domain.enums import Availability, PriceBasis
|
||||||
|
from app.domain.models import Budget, HistoryWindow, PriceObservation, PriceStatistics, ScoringModel
|
||||||
|
from app.scoring.mathutil import clamp
|
||||||
|
|
||||||
|
|
||||||
|
def total_or_none(observation: PriceObservation) -> Decimal | None:
|
||||||
|
return observation.total_price
|
||||||
|
|
||||||
|
|
||||||
|
def select_current_observation(
|
||||||
|
observations: Sequence[PriceObservation],
|
||||||
|
*,
|
||||||
|
as_of: datetime,
|
||||||
|
snapshot_cutoff: datetime,
|
||||||
|
) -> PriceObservation | None:
|
||||||
|
eligible = [
|
||||||
|
item
|
||||||
|
for item in observations
|
||||||
|
if item.observed_at <= as_of and item.ingested_at <= snapshot_cutoff
|
||||||
|
]
|
||||||
|
if not eligible:
|
||||||
|
return None
|
||||||
|
return max(eligible, key=lambda item: (item.observed_at, item.ingested_at, item.id))
|
||||||
|
|
||||||
|
|
||||||
|
def median(values: Sequence[Decimal]) -> Decimal | None:
|
||||||
|
if not values:
|
||||||
|
return None
|
||||||
|
ordered = sorted(values)
|
||||||
|
mid = len(ordered) // 2
|
||||||
|
if len(ordered) % 2:
|
||||||
|
return ordered[mid]
|
||||||
|
return (ordered[mid - 1] + ordered[mid]) / Decimal("2")
|
||||||
|
|
||||||
|
|
||||||
|
def _quality_ok(observation: PriceObservation, minimum_confidence: Decimal) -> bool:
|
||||||
|
return (
|
||||||
|
observation.availability == Availability.IN_STOCK
|
||||||
|
and observation.total_price is not None
|
||||||
|
and observation.confidence >= minimum_confidence
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def daily_series(
|
||||||
|
observations: Sequence[PriceObservation],
|
||||||
|
*,
|
||||||
|
window_start: date,
|
||||||
|
window_end_exclusive: date,
|
||||||
|
minimum_confidence: Decimal,
|
||||||
|
) -> tuple[tuple[Decimal, ...], tuple[UUID, ...]]:
|
||||||
|
by_day: dict[date, PriceObservation] = {}
|
||||||
|
for item in observations:
|
||||||
|
day = item.observed_at.date()
|
||||||
|
if day < window_start or day >= window_end_exclusive:
|
||||||
|
continue
|
||||||
|
current = by_day.get(day)
|
||||||
|
if current is None or (item.observed_at, item.ingested_at, item.id) > (
|
||||||
|
current.observed_at,
|
||||||
|
current.ingested_at,
|
||||||
|
current.id,
|
||||||
|
):
|
||||||
|
by_day[day] = item
|
||||||
|
values: list[Decimal] = []
|
||||||
|
ids: list[UUID] = []
|
||||||
|
for day in sorted(by_day):
|
||||||
|
observation = by_day[day]
|
||||||
|
if _quality_ok(observation, minimum_confidence):
|
||||||
|
values.append(observation.total_price or Decimal("0"))
|
||||||
|
ids.append(observation.id)
|
||||||
|
return tuple(values), tuple(ids)
|
||||||
|
|
||||||
|
|
||||||
|
def build_offer_reference_history(
|
||||||
|
observations: Sequence[PriceObservation],
|
||||||
|
current: PriceObservation,
|
||||||
|
model: ScoringModel,
|
||||||
|
) -> dict[int, HistoryWindow]:
|
||||||
|
current_day = current.observed_at.date()
|
||||||
|
windows: dict[int, HistoryWindow] = {}
|
||||||
|
for days in (30, 90, 365):
|
||||||
|
values, ids = daily_series(
|
||||||
|
observations,
|
||||||
|
window_start=current_day - timedelta(days=days),
|
||||||
|
window_end_exclusive=current_day,
|
||||||
|
minimum_confidence=model.minimum_confidence,
|
||||||
|
)
|
||||||
|
windows[days] = HistoryWindow(days=days, values=values, observation_ids=ids)
|
||||||
|
return windows
|
||||||
|
|
||||||
|
|
||||||
|
def score_price(
|
||||||
|
*,
|
||||||
|
current_price: Decimal,
|
||||||
|
reference_history: Sequence[Decimal],
|
||||||
|
budget: Budget,
|
||||||
|
model: ScoringModel,
|
||||||
|
) -> tuple[Decimal, Decimal, Decimal | None, PriceBasis, str | None]:
|
||||||
|
"""Pure price score over current_price, reference_history and budget."""
|
||||||
|
if current_price <= budget.target:
|
||||||
|
budget_score = Decimal("100")
|
||||||
|
else:
|
||||||
|
budget_score = clamp(
|
||||||
|
Decimal("100") * (budget.maximum - current_price) / (budget.maximum - budget.target)
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(reference_history) < model.minimum_history_days:
|
||||||
|
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
|
||||||
|
|
||||||
|
reference = median(reference_history)
|
||||||
|
if reference is None or reference == 0:
|
||||||
|
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
|
||||||
|
|
||||||
|
discount = (reference - current_price) / reference
|
||||||
|
history_score = clamp(
|
||||||
|
Decimal("50") + Decimal("50") * discount / model.history_discount_for_max_score
|
||||||
|
)
|
||||||
|
price_score = model.budget_weight * budget_score + model.history_weight * history_score
|
||||||
|
return price_score, budget_score, history_score, PriceBasis.BUDGET_AND_HISTORY, None
|
||||||
|
|
||||||
|
|
||||||
|
def price_statistics_from_windows(
|
||||||
|
current: PriceObservation | None,
|
||||||
|
windows: dict[int, HistoryWindow],
|
||||||
|
) -> PriceStatistics:
|
||||||
|
window_30 = windows.get(30, HistoryWindow(30, (), ()))
|
||||||
|
window_90 = windows.get(90, HistoryWindow(90, (), ()))
|
||||||
|
window_365 = windows.get(365, HistoryWindow(365, (), ()))
|
||||||
|
used = window_30.observation_ids + window_90.observation_ids + window_365.observation_ids
|
||||||
|
if current is not None:
|
||||||
|
used = (current.id,) + used
|
||||||
|
lowest = min(window_365.values) if window_365.values else None
|
||||||
|
return PriceStatistics(
|
||||||
|
current_total=None if current is None else current.total_price,
|
||||||
|
median_30d=median(window_30.values),
|
||||||
|
median_30d_days=window_30.day_count,
|
||||||
|
median_90d=median(window_90.values),
|
||||||
|
median_90d_days=window_90.day_count,
|
||||||
|
lowest_365d=lowest,
|
||||||
|
lowest_365d_days=window_365.day_count,
|
||||||
|
lowest_365d_is_full_year=window_365.day_count >= 365,
|
||||||
|
used_observation_ids=tuple(dict.fromkeys(used)),
|
||||||
|
current_observation_id=None if current is None else current.id,
|
||||||
|
)
|
||||||
86
backend/app/scoring/product.py
Normal file
86
backend/app/scoring/product.py
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
from app.domain.attributes import statement_is_usable
|
||||||
|
from app.domain.enums import ConstraintStatus, PreferenceDirection
|
||||||
|
from app.domain.models import (
|
||||||
|
AttributeStatement,
|
||||||
|
ConstraintResult,
|
||||||
|
Preference,
|
||||||
|
ScoreComponent,
|
||||||
|
ScoringModel,
|
||||||
|
)
|
||||||
|
from app.scoring.mathutil import as_decimal, clamp
|
||||||
|
|
||||||
|
|
||||||
|
def score_numeric_linear(value: Decimal, acceptable: Decimal, ideal: Decimal, direction: PreferenceDirection) -> Decimal:
|
||||||
|
if direction == PreferenceDirection.HIGHER:
|
||||||
|
return clamp(Decimal("50") + Decimal("50") * (value - acceptable) / (ideal - acceptable))
|
||||||
|
return clamp(Decimal("50") + Decimal("50") * (acceptable - value) / (acceptable - ideal))
|
||||||
|
|
||||||
|
|
||||||
|
def score_product(
|
||||||
|
preferences: tuple[Preference, ...],
|
||||||
|
snapshot: dict[str, AttributeStatement],
|
||||||
|
constraint_results: tuple[ConstraintResult, ...],
|
||||||
|
model: ScoringModel,
|
||||||
|
) -> tuple[Decimal, Decimal, tuple[ScoreComponent, ...]]:
|
||||||
|
components: list[ScoreComponent] = []
|
||||||
|
weighted = Decimal("0")
|
||||||
|
total_weight = sum((pref.weight for pref in preferences), Decimal("0"))
|
||||||
|
usable_weight = Decimal("0")
|
||||||
|
|
||||||
|
for preference in preferences:
|
||||||
|
statement = snapshot.get(preference.attribute)
|
||||||
|
usable = statement_is_usable(statement, model.minimum_confidence)
|
||||||
|
if usable:
|
||||||
|
raw = score_numeric_linear(
|
||||||
|
as_decimal(statement.normalized_value),
|
||||||
|
preference.acceptable,
|
||||||
|
preference.ideal,
|
||||||
|
preference.direction,
|
||||||
|
)
|
||||||
|
usable_weight += preference.weight
|
||||||
|
else:
|
||||||
|
raw = model.missing_preference_score
|
||||||
|
contribution = preference.weight * raw
|
||||||
|
weighted += contribution
|
||||||
|
components.append(
|
||||||
|
ScoreComponent(
|
||||||
|
kind="preference",
|
||||||
|
component_key=preference.id,
|
||||||
|
raw_score=raw,
|
||||||
|
weight=preference.weight,
|
||||||
|
contribution=contribution,
|
||||||
|
details={
|
||||||
|
"attribute": preference.attribute,
|
||||||
|
"strategy": preference.strategy.value,
|
||||||
|
"direction": preference.direction.value,
|
||||||
|
"acceptable": preference.acceptable,
|
||||||
|
"ideal": preference.ideal,
|
||||||
|
"usable": usable,
|
||||||
|
"attribute_row_id": None if statement is None else str(statement.id),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
base = weighted / total_weight if total_weight else Decimal("0")
|
||||||
|
optional_issues = [
|
||||||
|
result
|
||||||
|
for result in constraint_results
|
||||||
|
if not result.mandatory and result.status in {ConstraintStatus.FAIL.value, ConstraintStatus.UNKNOWN.value}
|
||||||
|
]
|
||||||
|
penalty = model.optional_constraint_penalty_points * Decimal(len(optional_issues))
|
||||||
|
product_score = clamp(base - penalty)
|
||||||
|
if optional_issues:
|
||||||
|
components.append(
|
||||||
|
ScoreComponent(
|
||||||
|
kind="optional_penalty",
|
||||||
|
component_key="optional_constraints",
|
||||||
|
raw_score=penalty,
|
||||||
|
weight=None,
|
||||||
|
contribution=-penalty,
|
||||||
|
details={"count": len(optional_issues), "constraint_ids": [item.constraint_id for item in optional_issues]},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
coverage = (usable_weight / total_weight) if total_weight else Decimal("0")
|
||||||
|
return product_score, coverage, tuple(components)
|
||||||
0
backend/app/services/__init__.py
Normal file
0
backend/app/services/__init__.py
Normal file
224
backend/app/services/catalog_service.py
Normal file
224
backend/app/services/catalog_service.py
Normal file
|
|
@ -0,0 +1,224 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.discovery.enums import FilterType, MappingStatus
|
||||||
|
from app.domain.serialize import jsonable
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.persistence.discovery_models import (
|
||||||
|
CriterionDefinitionRecord,
|
||||||
|
CriterionMappingRecord,
|
||||||
|
ProductCategoryRecord,
|
||||||
|
SourceCategoryRecord,
|
||||||
|
SourceFilterDefinitionRecord,
|
||||||
|
SourceFilterOptionRecord,
|
||||||
|
SourceFilterRangeRecord,
|
||||||
|
)
|
||||||
|
from app.discovery.definitions import upsert_definition
|
||||||
|
from app.persistence.repositories import CatalogRepository
|
||||||
|
|
||||||
|
|
||||||
|
FILTER_OPERATORS = {
|
||||||
|
FilterType.BOOLEAN.value: ["=", "!="],
|
||||||
|
FilterType.ENUM.value: ["=", "!="],
|
||||||
|
FilterType.MULTI_ENUM.value: ["in", "not_in"],
|
||||||
|
FilterType.NUMERIC_RANGE.value: [">=", "<=", "between"],
|
||||||
|
FilterType.NUMERIC_THRESHOLD.value: [">=", ">", "<=", "<"],
|
||||||
|
FilterType.TEXT.value: ["=", "contains"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CatalogService:
|
||||||
|
def __init__(self, session: Session) -> None:
|
||||||
|
self.session = session
|
||||||
|
self.catalog = CatalogRepository(session)
|
||||||
|
|
||||||
|
def list_product_categories(self) -> list[dict[str, Any]]:
|
||||||
|
rows = self.session.scalars(select(ProductCategoryRecord).order_by(ProductCategoryRecord.name)).all()
|
||||||
|
return [{"id": row.id, "name": row.name, "status": row.status} for row in rows]
|
||||||
|
|
||||||
|
def get_product_category(self, category_id: str) -> ProductCategoryRecord:
|
||||||
|
row = self.session.get(ProductCategoryRecord, category_id)
|
||||||
|
if row is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "product category not found", {"id": category_id})
|
||||||
|
return row
|
||||||
|
|
||||||
|
def sources_for_category(self, product_category_id: str) -> list[dict[str, Any]]:
|
||||||
|
self.get_product_category(product_category_id)
|
||||||
|
rows = self.session.scalars(
|
||||||
|
select(SourceCategoryRecord)
|
||||||
|
.where(SourceCategoryRecord.product_category_id == product_category_id)
|
||||||
|
.order_by(SourceCategoryRecord.source_id)
|
||||||
|
).all()
|
||||||
|
sources = {item.id: item for item in self.catalog.list_sources()}
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"source": row.source_id,
|
||||||
|
"source_category_id": row.id,
|
||||||
|
"display_name": sources[row.source_id].name if row.source_id in sources else row.name,
|
||||||
|
"external_category_key": row.external_category_key,
|
||||||
|
"status": row.status,
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
def source_categories(self, source_id: str) -> list[dict[str, Any]]:
|
||||||
|
if self.catalog.get_source(source_id) is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||||
|
rows = self.session.scalars(
|
||||||
|
select(SourceCategoryRecord)
|
||||||
|
.where(SourceCategoryRecord.source_id == source_id)
|
||||||
|
.order_by(SourceCategoryRecord.external_category_key)
|
||||||
|
).all()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": row.id,
|
||||||
|
"source": row.source_id,
|
||||||
|
"source_category_id": row.id,
|
||||||
|
"external_category_key": row.external_category_key,
|
||||||
|
"name": row.name,
|
||||||
|
"product_category_id": row.product_category_id,
|
||||||
|
"status": row.status,
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
def available_criteria(self, product_category_id: str, source_id: str) -> dict[str, Any]:
|
||||||
|
self.get_product_category(product_category_id)
|
||||||
|
if self.catalog.get_source(source_id) is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||||
|
source_category = self.session.scalars(
|
||||||
|
select(SourceCategoryRecord).where(
|
||||||
|
SourceCategoryRecord.source_id == source_id,
|
||||||
|
SourceCategoryRecord.product_category_id == product_category_id,
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
criteria: list[dict[str, Any]] = []
|
||||||
|
if source_category is not None:
|
||||||
|
filters = self.session.scalars(
|
||||||
|
select(SourceFilterDefinitionRecord)
|
||||||
|
.where(SourceFilterDefinitionRecord.source_category_id == source_category.id)
|
||||||
|
.order_by(SourceFilterDefinitionRecord.label)
|
||||||
|
).all()
|
||||||
|
mappings = {
|
||||||
|
row.external_filter_key: row
|
||||||
|
for row in self.session.scalars(
|
||||||
|
select(CriterionMappingRecord).where(
|
||||||
|
CriterionMappingRecord.source_category_id == source_category.id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for filt in filters:
|
||||||
|
mapping = mappings.get(filt.external_filter_key)
|
||||||
|
definition = None if mapping is None else self.session.get(CriterionDefinitionRecord, mapping.criterion_definition_id)
|
||||||
|
if definition is None:
|
||||||
|
definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
{
|
||||||
|
"key": f"{source_id}:{source_category.external_category_key}:{filt.external_filter_key}",
|
||||||
|
"origin": "SOURCE",
|
||||||
|
"source": source_id,
|
||||||
|
"product_category_id": product_category_id,
|
||||||
|
"label": filt.label,
|
||||||
|
"value_type": "number"
|
||||||
|
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
|
||||||
|
else "string",
|
||||||
|
"unit": filt.canonical_unit,
|
||||||
|
"category": product_category_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.session.flush()
|
||||||
|
options = self.session.scalars(
|
||||||
|
select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||||
|
).all()
|
||||||
|
rng = self.session.get(SourceFilterRangeRecord, filt.id)
|
||||||
|
criteria.append(
|
||||||
|
self._criterion_out(
|
||||||
|
definition,
|
||||||
|
filt,
|
||||||
|
mapping,
|
||||||
|
options,
|
||||||
|
rng,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
internals = self.session.scalars(
|
||||||
|
select(CriterionDefinitionRecord)
|
||||||
|
.where(
|
||||||
|
CriterionDefinitionRecord.origin == "INTERNAL",
|
||||||
|
(CriterionDefinitionRecord.product_category_id == product_category_id)
|
||||||
|
| (CriterionDefinitionRecord.category == product_category_id),
|
||||||
|
)
|
||||||
|
.order_by(CriterionDefinitionRecord.label)
|
||||||
|
).all()
|
||||||
|
seen = {item["criterion_definition_id"] for item in criteria}
|
||||||
|
for definition in internals:
|
||||||
|
if definition.id in seen:
|
||||||
|
continue
|
||||||
|
criteria.append(self._criterion_out(definition, None, None, [], None))
|
||||||
|
return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria})
|
||||||
|
|
||||||
|
def list_templates(self) -> list[dict[str, Any]]:
|
||||||
|
rows, _total = self.catalog.list_profiles(200, 0)
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": row.id,
|
||||||
|
"version": row.version,
|
||||||
|
"name": row.definition.get("name") or row.id,
|
||||||
|
"product_category_id": row.definition.get("category"),
|
||||||
|
"market": row.definition.get("market"),
|
||||||
|
"currency": row.definition.get("currency"),
|
||||||
|
"description": row.definition.get("description"),
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
def _criterion_out(
|
||||||
|
self,
|
||||||
|
definition: CriterionDefinitionRecord,
|
||||||
|
filt: SourceFilterDefinitionRecord | None,
|
||||||
|
mapping: CriterionMappingRecord | None,
|
||||||
|
options: list[SourceFilterOptionRecord],
|
||||||
|
rng: SourceFilterRangeRecord | None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
filter_type = None if filt is None else filt.filter_type
|
||||||
|
return {
|
||||||
|
"criterion_definition_id": definition.id,
|
||||||
|
"label": definition.label,
|
||||||
|
"description": definition.description,
|
||||||
|
"origin": definition.origin,
|
||||||
|
"source": definition.source,
|
||||||
|
"product_category_id": definition.product_category_id,
|
||||||
|
"filter_type": filter_type,
|
||||||
|
"unit": definition.unit or (None if filt is None else filt.canonical_unit),
|
||||||
|
"operators": FILTER_OPERATORS.get(filter_type or "", ["=", ">=", "<="]),
|
||||||
|
"options": [
|
||||||
|
{
|
||||||
|
"source_value": option.source_value,
|
||||||
|
"normalized_value": option.normalized_value,
|
||||||
|
"operator": option.operator,
|
||||||
|
"unit": option.unit,
|
||||||
|
}
|
||||||
|
for option in options
|
||||||
|
],
|
||||||
|
"range": None
|
||||||
|
if rng is None
|
||||||
|
else {"minimum": rng.minimum, "maximum": rng.maximum, "step": rng.step, "unit": rng.unit},
|
||||||
|
"mapping_status": None if mapping is None else mapping.status,
|
||||||
|
"supports_discovery": mapping is not None
|
||||||
|
and mapping.status == MappingStatus.VERIFIED.value
|
||||||
|
and filt is not None,
|
||||||
|
"scoring_attribute_key": definition.scoring_attribute_key,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_product_category(session: Session, category_id: str, name: str | None = None) -> ProductCategoryRecord:
|
||||||
|
row = session.get(ProductCategoryRecord, category_id)
|
||||||
|
if row is not None:
|
||||||
|
return row
|
||||||
|
row = ProductCategoryRecord(id=category_id, name=name or category_id.title(), status="ACTIVE")
|
||||||
|
session.add(row)
|
||||||
|
session.flush()
|
||||||
|
return row
|
||||||
1431
backend/app/services/discovery_service.py
Normal file
1431
backend/app/services/discovery_service.py
Normal file
File diff suppressed because it is too large
Load Diff
210
backend/app/services/evaluation_service.py
Normal file
210
backend/app/services/evaluation_service.py
Normal file
|
|
@ -0,0 +1,210 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.clock import now_utc
|
||||||
|
from app.domain.serialize import jsonable
|
||||||
|
from app.domain.timeutil import require_utc, to_iso_z
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.persistence import models as orm
|
||||||
|
from app.persistence.db import engine
|
||||||
|
from app.persistence.repositories import CatalogRepository
|
||||||
|
from app.scoring.engine import evaluate_deal
|
||||||
|
from app.scoring.mathutil import round_score
|
||||||
|
from app.settings import get_settings
|
||||||
|
from sqlalchemy.orm import Session as SASession
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_and_store(
|
||||||
|
*,
|
||||||
|
variant_id: UUID,
|
||||||
|
payload: dict[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
now = now_utc()
|
||||||
|
as_of = payload.get("as_of")
|
||||||
|
if as_of is None:
|
||||||
|
as_of = now
|
||||||
|
else:
|
||||||
|
as_of = require_utc(as_of, "as_of")
|
||||||
|
if as_of > now:
|
||||||
|
raise AppError(422, "AS_OF_IN_FUTURE", "as_of must not be in the future")
|
||||||
|
snapshot_cutoff = now
|
||||||
|
|
||||||
|
maker = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||||
|
with engine.connect().execution_options(isolation_level="REPEATABLE READ") as connection:
|
||||||
|
with SASession(bind=connection) as session:
|
||||||
|
with session.begin():
|
||||||
|
repo = CatalogRepository(session)
|
||||||
|
variant = repo.get_variant(variant_id)
|
||||||
|
if variant is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||||
|
offer = repo.get_offer(payload["offer_id"])
|
||||||
|
if offer is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(payload["offer_id"])})
|
||||||
|
if offer.variant_id != variant_id:
|
||||||
|
raise AppError(422, "OFFER_VARIANT_MISMATCH", "offer does not belong to the variant")
|
||||||
|
product = repo.get_product(variant.product_id)
|
||||||
|
if product is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "product not found")
|
||||||
|
profile = repo.get_profile(payload["profile_id"], int(payload["profile_version"]))
|
||||||
|
if profile is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "profile version not found")
|
||||||
|
model = repo.get_scoring_model(profile.scoring_version)
|
||||||
|
if model is None:
|
||||||
|
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version is missing")
|
||||||
|
attributes = tuple(repo.list_attributes(variant_id))
|
||||||
|
observations, _ = repo.list_observations(offer.id, limit=None)
|
||||||
|
result = evaluate_deal(
|
||||||
|
profile=profile,
|
||||||
|
scoring_model=model,
|
||||||
|
product=product,
|
||||||
|
variant=variant,
|
||||||
|
offer=offer,
|
||||||
|
attributes=attributes,
|
||||||
|
observations=tuple(observations),
|
||||||
|
definitions=repo.all_definitions(),
|
||||||
|
as_of=as_of,
|
||||||
|
snapshot_cutoff=snapshot_cutoff,
|
||||||
|
engine_build=get_settings().engine_build,
|
||||||
|
)
|
||||||
|
response = evaluation_to_response(result, evaluation_id=uuid4(), created_at=now)
|
||||||
|
record = orm.EvaluationRecord(
|
||||||
|
id=response["id"],
|
||||||
|
variant_id=variant.id,
|
||||||
|
offer_id=offer.id,
|
||||||
|
profile_id=profile.id,
|
||||||
|
profile_version=profile.version,
|
||||||
|
scoring_version=model.version,
|
||||||
|
as_of=as_of,
|
||||||
|
snapshot_cutoff=snapshot_cutoff,
|
||||||
|
scoring_hash=model.definition_hash,
|
||||||
|
engine_build=result.engine_build,
|
||||||
|
product_eligibility=result.product_eligibility.value,
|
||||||
|
offer_eligibility=result.offer_eligibility.value,
|
||||||
|
overall_eligibility=result.overall_eligibility.value,
|
||||||
|
product_score=result.product_score,
|
||||||
|
price_score=result.price_score,
|
||||||
|
deal_score=result.deal_score,
|
||||||
|
classification=result.classification,
|
||||||
|
price_reference_scope=result.price_reference_scope.value,
|
||||||
|
input_snapshot=result.input_snapshot,
|
||||||
|
result_snapshot=response,
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
repo.add_evaluation(record)
|
||||||
|
for component in result.components:
|
||||||
|
repo.add_component(
|
||||||
|
orm.EvaluationComponentRecord(
|
||||||
|
id=uuid4(),
|
||||||
|
evaluation_id=record.id,
|
||||||
|
kind=component.kind,
|
||||||
|
component_key=component.component_key,
|
||||||
|
raw_score=component.raw_score,
|
||||||
|
weight=component.weight,
|
||||||
|
contribution=component.contribution,
|
||||||
|
details=jsonable(component.details),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def load_evaluation(session: Session, evaluation_id: UUID) -> dict[str, Any]:
|
||||||
|
record = CatalogRepository(session).get_evaluation(evaluation_id)
|
||||||
|
if record is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "evaluation not found", {"id": str(evaluation_id)})
|
||||||
|
return record.result_snapshot
|
||||||
|
|
||||||
|
|
||||||
|
def evaluation_to_response(result, *, evaluation_id: UUID, created_at: datetime) -> dict[str, Any]:
|
||||||
|
stats = result.price_statistics
|
||||||
|
return {
|
||||||
|
"id": str(evaluation_id),
|
||||||
|
"profile_id": result.profile_id,
|
||||||
|
"profile_version": result.profile_version,
|
||||||
|
"scoring_version": result.scoring_version,
|
||||||
|
"variant_id": str(result.variant_id),
|
||||||
|
"offer_id": str(result.offer_id),
|
||||||
|
"as_of": to_iso_z(result.as_of),
|
||||||
|
"snapshot_cutoff": to_iso_z(result.snapshot_cutoff),
|
||||||
|
"scoring_hash": result.scoring_hash,
|
||||||
|
"engine_build": result.engine_build,
|
||||||
|
"product_eligibility": result.product_eligibility.value,
|
||||||
|
"offer_eligibility": result.offer_eligibility.value,
|
||||||
|
"overall_eligibility": result.overall_eligibility.value,
|
||||||
|
"eligible": result.eligible,
|
||||||
|
"product_score": _num(result.product_score),
|
||||||
|
"price_score": _num(result.price_score),
|
||||||
|
"deal_score": _num(result.deal_score),
|
||||||
|
"classification": result.classification,
|
||||||
|
"price_basis": None if result.price_basis is None else result.price_basis.value,
|
||||||
|
"product_coverage": _num(result.product_coverage),
|
||||||
|
"failed_constraints": [_constraint(item) for item in result.failed_constraints],
|
||||||
|
"unknown_constraints": [_constraint(item) for item in result.unknown_constraints],
|
||||||
|
"constraint_results": [_constraint(item) for item in result.constraint_results],
|
||||||
|
"product_gates": [_gate(item) for item in result.product_gates],
|
||||||
|
"offer_gates": [_gate(item) for item in result.offer_gates],
|
||||||
|
"components": [_component(item) for item in result.components],
|
||||||
|
"price_statistics": {
|
||||||
|
"current_total": _money(stats.current_total),
|
||||||
|
"median_30d": _money(stats.median_30d),
|
||||||
|
"median_30d_days": stats.median_30d_days,
|
||||||
|
"median_90d": _money(stats.median_90d),
|
||||||
|
"median_90d_days": stats.median_90d_days,
|
||||||
|
"lowest_365d": _money(stats.lowest_365d),
|
||||||
|
"lowest_365d_days": stats.lowest_365d_days,
|
||||||
|
"lowest_365d_is_full_year": stats.lowest_365d_is_full_year,
|
||||||
|
"used_observation_ids": [str(item) for item in stats.used_observation_ids],
|
||||||
|
"current_observation_id": None if stats.current_observation_id is None else str(stats.current_observation_id),
|
||||||
|
},
|
||||||
|
"price_reference_scope": result.price_reference_scope.value,
|
||||||
|
"reasons": [{"code": item.code, "message": item.message} for item in result.reasons],
|
||||||
|
"warnings": list(result.warnings),
|
||||||
|
"input_snapshot": result.input_snapshot,
|
||||||
|
"created_at": to_iso_z(created_at),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _num(value: Decimal | None) -> float | None:
|
||||||
|
return None if value is None else float(round_score(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _money(value: Decimal | None) -> str | None:
|
||||||
|
return None if value is None else format(value.quantize(Decimal("0.01")), "f")
|
||||||
|
|
||||||
|
|
||||||
|
def _constraint(item) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"constraint_id": item.constraint_id,
|
||||||
|
"attribute": item.attribute,
|
||||||
|
"operator": item.operator,
|
||||||
|
"expected": jsonable(item.expected),
|
||||||
|
"actual": jsonable(item.actual),
|
||||||
|
"status": item.status,
|
||||||
|
"mandatory": item.mandatory,
|
||||||
|
"attribute_row_id": None if item.attribute_row_id is None else str(item.attribute_row_id),
|
||||||
|
"reason_code": item.reason_code,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _gate(item) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"key": item.key,
|
||||||
|
"status": item.status,
|
||||||
|
"reason_code": item.reason_code,
|
||||||
|
"details": jsonable(item.details),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _component(item) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"kind": item.kind,
|
||||||
|
"component_key": item.component_key,
|
||||||
|
"raw_score": None if item.raw_score is None else float(round_score(item.raw_score)),
|
||||||
|
"weight": None if item.weight is None else float(item.weight),
|
||||||
|
"contribution": None if item.contribution is None else float(round_score(item.contribution)),
|
||||||
|
"details": jsonable(item.details),
|
||||||
|
}
|
||||||
94
backend/app/services/import_service.py
Normal file
94
backend/app/services/import_service.py
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.services.ingestion_service import IngestionService
|
||||||
|
from app.sources.base import ProductSource
|
||||||
|
|
||||||
|
|
||||||
|
class ImportService:
|
||||||
|
def __init__(self, ingestion: IngestionService) -> None:
|
||||||
|
self.ingestion = ingestion
|
||||||
|
|
||||||
|
def import_product(self, source: ProductSource, external_id: str) -> None:
|
||||||
|
product = source.get_product(external_id)
|
||||||
|
try:
|
||||||
|
self.ingestion.create_product(
|
||||||
|
{
|
||||||
|
"id": product.id,
|
||||||
|
"category": product.category,
|
||||||
|
"brand": product.brand,
|
||||||
|
"name": product.name,
|
||||||
|
"model": product.model,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
if exc.status_code != 409:
|
||||||
|
raise
|
||||||
|
for variant in product.variants:
|
||||||
|
try:
|
||||||
|
self.ingestion.create_variant(
|
||||||
|
product.id,
|
||||||
|
{
|
||||||
|
"id": variant.id,
|
||||||
|
"name": variant.name,
|
||||||
|
"manufacturer_sku": variant.manufacturer_sku,
|
||||||
|
"identifiers": variant.identifiers,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
if exc.status_code != 409:
|
||||||
|
raise
|
||||||
|
for attribute in variant.attributes:
|
||||||
|
try:
|
||||||
|
self.ingestion.add_attribute(
|
||||||
|
variant.id,
|
||||||
|
{
|
||||||
|
"id": attribute.id,
|
||||||
|
"attribute": attribute.attribute,
|
||||||
|
"raw_value": attribute.raw_value,
|
||||||
|
"normalized_value": attribute.normalized_value,
|
||||||
|
"unit": attribute.unit,
|
||||||
|
"normalization_status": attribute.normalization_status,
|
||||||
|
"normalizer_version": attribute.normalizer_version,
|
||||||
|
"confidence": attribute.confidence,
|
||||||
|
"source_id": source.source_key,
|
||||||
|
"source_reference": attribute.source_reference,
|
||||||
|
"observed_at": attribute.observed_at,
|
||||||
|
"normalization_error": attribute.normalization_error,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
if exc.status_code != 409:
|
||||||
|
raise
|
||||||
|
for offer in source.get_offers(external_id):
|
||||||
|
try:
|
||||||
|
self.ingestion.create_offer(
|
||||||
|
offer.variant_id,
|
||||||
|
{
|
||||||
|
"id": offer.id,
|
||||||
|
"source_id": source.source_key,
|
||||||
|
"external_id": offer.external_id,
|
||||||
|
"seller": offer.seller,
|
||||||
|
"url": offer.url,
|
||||||
|
"market": offer.market,
|
||||||
|
"currency": offer.currency,
|
||||||
|
"condition": offer.condition,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
if exc.status_code != 409:
|
||||||
|
raise
|
||||||
|
for price in source.get_price_history(offer.external_id):
|
||||||
|
self.ingestion.add_price(
|
||||||
|
offer.id,
|
||||||
|
{
|
||||||
|
"id": price.id,
|
||||||
|
"source_id": source.source_key,
|
||||||
|
"external_observation_id": price.external_observation_id,
|
||||||
|
"observed_at": price.observed_at,
|
||||||
|
"item_price": price.item_price,
|
||||||
|
"shipping_price": price.shipping_price,
|
||||||
|
"currency": price.currency,
|
||||||
|
"availability": price.availability,
|
||||||
|
"raw_payload": price.raw_payload,
|
||||||
|
"confidence": price.confidence,
|
||||||
|
},
|
||||||
|
)
|
||||||
263
backend/app/services/ingestion_service.py
Normal file
263
backend/app/services/ingestion_service.py
Normal file
|
|
@ -0,0 +1,263 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
|
||||||
|
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
|
||||||
|
from app.clock import now_utc
|
||||||
|
from app.domain.timeutil import require_utc
|
||||||
|
from app.domain.validation import DomainValidationError, parse_profile, validate_attribute_key, validate_currency
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.normalization.manual import ManualNormalizer
|
||||||
|
from app.persistence import models as orm
|
||||||
|
from app.persistence.mapping import to_observation
|
||||||
|
from app.persistence.repositories import CatalogRepository
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> datetime:
|
||||||
|
return now_utc()
|
||||||
|
|
||||||
|
|
||||||
|
def _uuid(value: UUID | None) -> UUID:
|
||||||
|
return value or uuid4()
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionService:
|
||||||
|
def __init__(self, session: Session) -> None:
|
||||||
|
self.session = session
|
||||||
|
self.repo = CatalogRepository(session)
|
||||||
|
self.normalizer = ManualNormalizer()
|
||||||
|
|
||||||
|
def create_definition(self, payload: dict[str, Any]) -> AttributeDefinition:
|
||||||
|
key = str(payload["key"])
|
||||||
|
validate_attribute_key(key)
|
||||||
|
row = orm.AttributeDefinitionRecord(
|
||||||
|
key=key,
|
||||||
|
value_type=ValueType(payload["value_type"]).value,
|
||||||
|
canonical_unit=payload.get("canonical_unit"),
|
||||||
|
minimum=payload.get("minimum"),
|
||||||
|
maximum=payload.get("maximum"),
|
||||||
|
description=payload.get("description"),
|
||||||
|
)
|
||||||
|
if self.repo.get_definition(key) is not None:
|
||||||
|
raise AppError(409, "CONFLICT", "attribute definition already exists", {"key": key})
|
||||||
|
return self.repo.add_definition(row)
|
||||||
|
|
||||||
|
def create_profile(self, payload: dict[str, Any]) -> orm.ProductProfileRecord:
|
||||||
|
try:
|
||||||
|
model_versions = self.repo.scoring_versions()
|
||||||
|
scoring_version = payload.get("scoring_version")
|
||||||
|
model = self.repo.get_scoring_model(scoring_version) if scoring_version else None
|
||||||
|
if model is None:
|
||||||
|
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version does not exist")
|
||||||
|
profile = parse_profile(
|
||||||
|
payload,
|
||||||
|
attribute_definitions=self.repo.all_definitions(),
|
||||||
|
scoring_versions=model_versions,
|
||||||
|
supported_strategies=set(model.supported_preference_strategies),
|
||||||
|
)
|
||||||
|
except DomainValidationError as exc:
|
||||||
|
raise AppError(422, "PROFILE_INVALID", str(exc), exc.details) from exc
|
||||||
|
except AppError:
|
||||||
|
raise
|
||||||
|
row = orm.ProductProfileRecord(
|
||||||
|
id=profile.id,
|
||||||
|
version=profile.version,
|
||||||
|
name=profile.name,
|
||||||
|
category=profile.category,
|
||||||
|
scoring_version=profile.scoring_version,
|
||||||
|
definition=profile.definition,
|
||||||
|
definition_hash=profile.definition_hash,
|
||||||
|
created_at=_now(),
|
||||||
|
)
|
||||||
|
if self.repo.get_profile(profile.id, profile.version) is not None:
|
||||||
|
raise AppError(409, "CONFLICT", "profile version already exists", {"id": profile.id, "version": profile.version})
|
||||||
|
return self.repo.add_profile(row)
|
||||||
|
|
||||||
|
def create_product(self, payload: dict[str, Any]) -> Product:
|
||||||
|
product_id = _uuid(payload.get("id"))
|
||||||
|
if self.repo.get_product(product_id) is not None:
|
||||||
|
raise AppError(409, "CONFLICT", "product already exists", {"id": str(product_id)})
|
||||||
|
row = orm.ProductRecord(
|
||||||
|
id=product_id,
|
||||||
|
category=str(payload["category"]),
|
||||||
|
brand=str(payload["brand"]),
|
||||||
|
name=str(payload["name"]),
|
||||||
|
model=payload.get("model"),
|
||||||
|
created_at=_now(),
|
||||||
|
)
|
||||||
|
return self.repo.add_product(row)
|
||||||
|
|
||||||
|
def create_variant(self, product_id: UUID, payload: dict[str, Any]) -> ProductVariant:
|
||||||
|
if self.repo.get_product(product_id) is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
|
||||||
|
variant_id = _uuid(payload.get("id"))
|
||||||
|
if self.repo.get_variant(variant_id) is not None:
|
||||||
|
raise AppError(409, "CONFLICT", "variant already exists", {"id": str(variant_id)})
|
||||||
|
row = orm.ProductVariantRecord(
|
||||||
|
id=variant_id,
|
||||||
|
product_id=product_id,
|
||||||
|
name=str(payload["name"]),
|
||||||
|
manufacturer_sku=payload.get("manufacturer_sku"),
|
||||||
|
identifiers=payload.get("identifiers") or {},
|
||||||
|
created_at=_now(),
|
||||||
|
)
|
||||||
|
return self.repo.add_variant(row)
|
||||||
|
|
||||||
|
def add_attribute(self, variant_id: UUID, payload: dict[str, Any]) -> AttributeStatement:
|
||||||
|
variant = self.repo.get_variant(variant_id)
|
||||||
|
if variant is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||||
|
definition = self.repo.get_definition(payload["attribute"])
|
||||||
|
if definition is None:
|
||||||
|
raise AppError(422, "UNKNOWN_ATTRIBUTE", "attribute definition does not exist")
|
||||||
|
if self.repo.get_source(payload["source_id"]) is None:
|
||||||
|
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
|
||||||
|
observed_at = require_utc(payload["observed_at"], "observed_at")
|
||||||
|
ingested_at = _now()
|
||||||
|
if observed_at > ingested_at:
|
||||||
|
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
|
||||||
|
confidence = Decimal(str(payload["confidence"]))
|
||||||
|
if not Decimal("0") <= confidence <= Decimal("1"):
|
||||||
|
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
|
||||||
|
statement = AttributeStatement(
|
||||||
|
id=_uuid(payload.get("id")),
|
||||||
|
variant_id=variant_id,
|
||||||
|
attribute=definition.key,
|
||||||
|
raw_value=payload["raw_value"],
|
||||||
|
normalized_value=payload.get("normalized_value"),
|
||||||
|
unit=payload.get("unit"),
|
||||||
|
normalization_status=NormalizationStatus(payload["normalization_status"]),
|
||||||
|
normalizer_version=payload.get("normalizer_version") or self.normalizer.version,
|
||||||
|
confidence=confidence,
|
||||||
|
source_id=payload["source_id"],
|
||||||
|
source_reference=payload["source_reference"],
|
||||||
|
observed_at=observed_at,
|
||||||
|
ingested_at=ingested_at,
|
||||||
|
normalization_error=payload.get("normalization_error"),
|
||||||
|
)
|
||||||
|
statement = self.normalizer.validate(definition, statement)
|
||||||
|
if self.repo.get_attribute(statement.id) is not None:
|
||||||
|
raise AppError(409, "CONFLICT", "attribute statement already exists", {"id": str(statement.id)})
|
||||||
|
row = orm.VariantAttributeRecord(
|
||||||
|
id=statement.id,
|
||||||
|
variant_id=variant_id,
|
||||||
|
attribute_key=statement.attribute,
|
||||||
|
source_id=statement.source_id,
|
||||||
|
raw_value=statement.raw_value,
|
||||||
|
normalized_value=statement.normalized_value,
|
||||||
|
unit=statement.unit,
|
||||||
|
normalization_status=statement.normalization_status.value,
|
||||||
|
normalizer_version=statement.normalizer_version,
|
||||||
|
confidence=statement.confidence,
|
||||||
|
source_reference=statement.source_reference,
|
||||||
|
observed_at=statement.observed_at,
|
||||||
|
ingested_at=statement.ingested_at,
|
||||||
|
normalization_error=statement.normalization_error,
|
||||||
|
)
|
||||||
|
return self.repo.add_attribute(row)
|
||||||
|
|
||||||
|
def create_offer(self, variant_id: UUID, payload: dict[str, Any]) -> Offer:
|
||||||
|
if self.repo.get_variant(variant_id) is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||||
|
if self.repo.get_source(payload["source_id"]) is None:
|
||||||
|
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
|
||||||
|
offer_id = _uuid(payload.get("id"))
|
||||||
|
if self.repo.get_offer(offer_id) is not None:
|
||||||
|
raise AppError(409, "CONFLICT", "offer already exists", {"id": str(offer_id)})
|
||||||
|
if self.repo.get_offer_by_external(payload["source_id"], str(payload["external_id"])) is not None:
|
||||||
|
raise AppError(409, "CONFLICT", "offer source/external_id already exists")
|
||||||
|
row = orm.OfferRecord(
|
||||||
|
id=offer_id,
|
||||||
|
variant_id=variant_id,
|
||||||
|
source_id=payload["source_id"],
|
||||||
|
external_id=str(payload["external_id"]),
|
||||||
|
seller=str(payload["seller"]),
|
||||||
|
url=payload.get("url"),
|
||||||
|
market=str(payload["market"]),
|
||||||
|
currency=validate_currency(str(payload["currency"])),
|
||||||
|
condition=OfferCondition(payload["condition"]).value,
|
||||||
|
created_at=_now(),
|
||||||
|
)
|
||||||
|
return self.repo.add_offer(row)
|
||||||
|
|
||||||
|
def add_price(self, offer_id: UUID, payload: dict[str, Any]) -> tuple[PriceObservation, bool]:
|
||||||
|
offer = self.repo.get_offer(offer_id)
|
||||||
|
if offer is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
||||||
|
if payload["source_id"] != offer.source_id:
|
||||||
|
raise AppError(422, "SOURCE_MISMATCH", "source_id must match the offer")
|
||||||
|
currency = validate_currency(str(payload["currency"]))
|
||||||
|
if currency != offer.currency:
|
||||||
|
raise AppError(422, "CURRENCY_MISMATCH", "currency must match the offer")
|
||||||
|
observed_at = require_utc(payload["observed_at"], "observed_at")
|
||||||
|
ingested_at = _now()
|
||||||
|
if observed_at > ingested_at:
|
||||||
|
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
|
||||||
|
item_price = Decimal(str(payload["item_price"]))
|
||||||
|
if item_price <= 0:
|
||||||
|
raise AppError(422, "ITEM_PRICE", "item_price must be > 0")
|
||||||
|
shipping = payload.get("shipping_price")
|
||||||
|
shipping_price = None if shipping is None else Decimal(str(shipping))
|
||||||
|
if shipping_price is not None and shipping_price < 0:
|
||||||
|
raise AppError(422, "SHIPPING_PRICE", "shipping_price must be >= 0")
|
||||||
|
confidence = Decimal(str(payload["confidence"]))
|
||||||
|
if not Decimal("0") <= confidence <= Decimal("1"):
|
||||||
|
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
|
||||||
|
external_id = str(payload["external_observation_id"])
|
||||||
|
existing = self.repo.get_observation_by_external(offer_id, external_id)
|
||||||
|
incoming = {
|
||||||
|
"source_id": payload["source_id"],
|
||||||
|
"observed_at": observed_at,
|
||||||
|
"item_price": item_price,
|
||||||
|
"shipping_price": shipping_price,
|
||||||
|
"currency": currency,
|
||||||
|
"availability": Availability(payload["availability"]).value,
|
||||||
|
"raw_payload": payload["raw_payload"],
|
||||||
|
"confidence": confidence,
|
||||||
|
}
|
||||||
|
if existing is not None:
|
||||||
|
if _observation_content(existing) != incoming:
|
||||||
|
raise AppError(409, "CONFLICT", "price observation content differs")
|
||||||
|
return to_observation(existing), False
|
||||||
|
row = orm.PriceObservationRecord(
|
||||||
|
id=_uuid(payload.get("id")),
|
||||||
|
offer_id=offer_id,
|
||||||
|
source_id=payload["source_id"],
|
||||||
|
external_observation_id=external_id,
|
||||||
|
observed_at=observed_at,
|
||||||
|
ingested_at=ingested_at,
|
||||||
|
item_price=item_price,
|
||||||
|
shipping_price=shipping_price,
|
||||||
|
currency=currency,
|
||||||
|
availability=incoming["availability"],
|
||||||
|
raw_payload=payload["raw_payload"],
|
||||||
|
confidence=confidence,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return self.repo.add_observation(row), True
|
||||||
|
except IntegrityError:
|
||||||
|
self.session.rollback()
|
||||||
|
existing = self.repo.get_observation_by_external(offer_id, external_id)
|
||||||
|
if existing is None:
|
||||||
|
raise
|
||||||
|
if _observation_content(existing) != incoming:
|
||||||
|
raise AppError(409, "CONFLICT", "price observation content differs")
|
||||||
|
return to_observation(existing), False
|
||||||
|
|
||||||
|
|
||||||
|
def _observation_content(row: orm.PriceObservationRecord) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"source_id": row.source_id,
|
||||||
|
"observed_at": row.observed_at,
|
||||||
|
"item_price": Decimal(str(row.item_price)),
|
||||||
|
"shipping_price": None if row.shipping_price is None else Decimal(str(row.shipping_price)),
|
||||||
|
"currency": row.currency,
|
||||||
|
"availability": row.availability,
|
||||||
|
"raw_payload": row.raw_payload,
|
||||||
|
"confidence": Decimal(str(row.confidence)),
|
||||||
|
}
|
||||||
346
backend/app/services/project_service.py
Normal file
346
backend/app/services/project_service.py
Normal file
|
|
@ -0,0 +1,346 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.clock import now_utc
|
||||||
|
from app.discovery.criteria import criterion_payload, parse_project_criteria
|
||||||
|
from app.discovery.definitions import get_definition, get_definition_by_key, upsert_definition
|
||||||
|
from app.discovery.enums import CandidateStatus, DiffStatus, ProjectStatus, RankingStatus, SearchStatus
|
||||||
|
from app.domain.serialize import jsonable
|
||||||
|
from app.domain.validation import DomainValidationError
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.persistence.discovery_models import (
|
||||||
|
BuyingProjectCriteriaRecord,
|
||||||
|
BuyingProjectRecord,
|
||||||
|
DiscoverySearchRecord,
|
||||||
|
ProductCategoryRecord,
|
||||||
|
ProjectCandidateRecord,
|
||||||
|
ProjectCriterionRecord,
|
||||||
|
SearchRunItemRecord,
|
||||||
|
SearchRunRecord,
|
||||||
|
)
|
||||||
|
from app.persistence.repositories import CatalogRepository
|
||||||
|
from app.services.catalog_service import CatalogService, ensure_product_category
|
||||||
|
|
||||||
|
|
||||||
|
INACTIVE_SEARCH = {SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}
|
||||||
|
|
||||||
|
|
||||||
|
def _uuid(value: UUID | None) -> UUID:
|
||||||
|
return value or uuid4()
|
||||||
|
|
||||||
|
|
||||||
|
class ProjectService:
|
||||||
|
def __init__(self, session: Session) -> None:
|
||||||
|
self.session = session
|
||||||
|
self.catalog = CatalogRepository(session)
|
||||||
|
self.categories = CatalogService(session)
|
||||||
|
|
||||||
|
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
template_id = payload.get("template_id") or payload.get("profile_id")
|
||||||
|
template_version = payload.get("template_version") or payload.get("profile_version")
|
||||||
|
profile = None
|
||||||
|
if template_id is not None:
|
||||||
|
if template_version is None:
|
||||||
|
raise AppError(422, "VALIDATION_ERROR", "template_version is required when template_id is set")
|
||||||
|
profile = self.catalog.get_profile(str(template_id), int(template_version))
|
||||||
|
if profile is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "template version not found")
|
||||||
|
product_category_id = payload.get("product_category_id") or (None if profile is None else profile.category)
|
||||||
|
if not product_category_id:
|
||||||
|
raise AppError(422, "VALIDATION_ERROR", "product_category_id is required")
|
||||||
|
if self.session.get(ProductCategoryRecord, product_category_id) is None:
|
||||||
|
ensure_product_category(self.session, str(product_category_id))
|
||||||
|
market = payload.get("market") or (None if profile is None else profile.market)
|
||||||
|
currency = payload.get("currency") or (None if profile is None else profile.currency)
|
||||||
|
if not market or not currency:
|
||||||
|
raise AppError(422, "VALIDATION_ERROR", "market and currency are required")
|
||||||
|
target_budget = payload.get("target_budget")
|
||||||
|
if target_budget is None and profile is not None and getattr(profile, "budget", None) is not None:
|
||||||
|
target_budget = getattr(profile.budget, "target", None)
|
||||||
|
criteria_payload = payload.get("criteria")
|
||||||
|
if not criteria_payload:
|
||||||
|
raise AppError(422, "CRITERIA_REQUIRED", "criteria are required")
|
||||||
|
try:
|
||||||
|
parsed = parse_project_criteria(criteria_payload)
|
||||||
|
except DomainValidationError as exc:
|
||||||
|
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
|
||||||
|
shortlist_size = int(payload.get("shortlist_size") or 5)
|
||||||
|
if shortlist_size < 1:
|
||||||
|
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
|
||||||
|
now = now_utc()
|
||||||
|
project = BuyingProjectRecord(
|
||||||
|
id=_uuid(payload.get("id")),
|
||||||
|
name=str(payload["name"]),
|
||||||
|
status=ProjectStatus.ACTIVE.value,
|
||||||
|
workspace_id=payload.get("workspace_id"),
|
||||||
|
product_category_id=str(product_category_id),
|
||||||
|
market=str(market),
|
||||||
|
currency=str(currency),
|
||||||
|
target_budget=None if target_budget is None else Decimal(str(target_budget)),
|
||||||
|
template_id=None if profile is None else profile.id,
|
||||||
|
template_version=None if profile is None else profile.version,
|
||||||
|
shortlist_size=shortlist_size,
|
||||||
|
criteria_version=1,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
archived_at=None,
|
||||||
|
)
|
||||||
|
self.session.add(project)
|
||||||
|
self.session.flush()
|
||||||
|
self._store_criteria(project.id, 1, parsed, str(product_category_id), now)
|
||||||
|
return self.get(project.id)
|
||||||
|
|
||||||
|
def replace_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
project = self._require_project(project_id)
|
||||||
|
try:
|
||||||
|
parsed = parse_project_criteria(payload)
|
||||||
|
except DomainValidationError as exc:
|
||||||
|
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
|
||||||
|
current = self.latest_criteria(project_id)
|
||||||
|
version = (current.version if current is not None else 0) + 1
|
||||||
|
now = now_utc()
|
||||||
|
self._store_criteria(project.id, version, parsed, project.product_category_id, now)
|
||||||
|
project.criteria_version = version
|
||||||
|
project.updated_at = now
|
||||||
|
for search in self.session.scalars(
|
||||||
|
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
|
||||||
|
):
|
||||||
|
if search.status not in INACTIVE_SEARCH:
|
||||||
|
search.status = SearchStatus.OUT_OF_SYNC.value
|
||||||
|
search.updated_at = now
|
||||||
|
self.session.flush()
|
||||||
|
return self.get(project.id)
|
||||||
|
|
||||||
|
def add_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return self.replace_criteria(project_id, payload)
|
||||||
|
|
||||||
|
def patch(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
project = self._require_project(project_id)
|
||||||
|
allowed = {"name", "target_budget", "shortlist_size", "status"}
|
||||||
|
unknown = set(payload) - allowed
|
||||||
|
if unknown:
|
||||||
|
raise AppError(422, "VALIDATION_ERROR", "field cannot be patched", {"fields": sorted(unknown)})
|
||||||
|
if "name" in payload:
|
||||||
|
project.name = str(payload["name"])
|
||||||
|
if "target_budget" in payload:
|
||||||
|
project.target_budget = None if payload["target_budget"] is None else Decimal(str(payload["target_budget"]))
|
||||||
|
if "shortlist_size" in payload:
|
||||||
|
size = int(payload["shortlist_size"])
|
||||||
|
if size < 1:
|
||||||
|
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
|
||||||
|
project.shortlist_size = size
|
||||||
|
if "status" in payload:
|
||||||
|
status = str(payload["status"])
|
||||||
|
if status not in {item.value for item in ProjectStatus}:
|
||||||
|
raise AppError(422, "VALIDATION_ERROR", "invalid project status")
|
||||||
|
project.status = status
|
||||||
|
if status == ProjectStatus.ARCHIVED.value:
|
||||||
|
project.archived_at = now_utc()
|
||||||
|
project.updated_at = now_utc()
|
||||||
|
self.session.flush()
|
||||||
|
return self.get(project.id)
|
||||||
|
|
||||||
|
def get(self, project_id: UUID) -> dict[str, Any]:
|
||||||
|
project = self._require_project(project_id)
|
||||||
|
criteria = self.latest_criteria(project_id)
|
||||||
|
return self._out(project, criteria)
|
||||||
|
|
||||||
|
def get_criteria(self, project_id: UUID) -> dict[str, Any]:
|
||||||
|
project = self._require_project(project_id)
|
||||||
|
criteria = self.latest_criteria(project_id)
|
||||||
|
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
|
||||||
|
return jsonable(
|
||||||
|
{
|
||||||
|
"project_id": project.id,
|
||||||
|
"criteria_version": None if criteria is None else criteria.version,
|
||||||
|
"items": items,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def list(self, limit: int, offset: int) -> tuple[list[dict[str, Any]], int]:
|
||||||
|
total = self.session.scalar(select(func.count()).select_from(BuyingProjectRecord)) or 0
|
||||||
|
rows = self.session.scalars(
|
||||||
|
select(BuyingProjectRecord).order_by(BuyingProjectRecord.created_at).limit(limit).offset(offset)
|
||||||
|
).all()
|
||||||
|
return [self._summary(row) for row in rows], total
|
||||||
|
|
||||||
|
def latest_criteria(self, project_id: UUID) -> BuyingProjectCriteriaRecord | None:
|
||||||
|
return self.session.scalars(
|
||||||
|
select(BuyingProjectCriteriaRecord)
|
||||||
|
.where(BuyingProjectCriteriaRecord.project_id == project_id)
|
||||||
|
.order_by(BuyingProjectCriteriaRecord.version.desc())
|
||||||
|
).first()
|
||||||
|
|
||||||
|
def criteria_version(self, project_id: UUID, version: int) -> BuyingProjectCriteriaRecord | None:
|
||||||
|
return self.session.get(BuyingProjectCriteriaRecord, (project_id, version))
|
||||||
|
|
||||||
|
def criteria_items(self, project_id: UUID, version: int) -> list[dict[str, Any]]:
|
||||||
|
rows = self.session.scalars(
|
||||||
|
select(ProjectCriterionRecord)
|
||||||
|
.where(ProjectCriterionRecord.project_id == project_id, ProjectCriterionRecord.version == version)
|
||||||
|
.order_by(ProjectCriterionRecord.item_id)
|
||||||
|
).all()
|
||||||
|
return [criterion_payload(row) for row in rows]
|
||||||
|
|
||||||
|
def available_criteria(self, project_id: UUID) -> dict[str, Any]:
|
||||||
|
project = self._require_project(project_id)
|
||||||
|
sources = self.categories.sources_for_category(project.product_category_id)
|
||||||
|
source = sources[0]["source"] if sources else "geizhals"
|
||||||
|
return self.categories.available_criteria(project.product_category_id, source)
|
||||||
|
|
||||||
|
def _store_criteria(
|
||||||
|
self,
|
||||||
|
project_id: UUID,
|
||||||
|
version: int,
|
||||||
|
parsed: dict[str, Any],
|
||||||
|
category: str | None,
|
||||||
|
created_at,
|
||||||
|
) -> None:
|
||||||
|
self.session.add(
|
||||||
|
BuyingProjectCriteriaRecord(
|
||||||
|
project_id=project_id,
|
||||||
|
version=version,
|
||||||
|
definition_hash=parsed["definition_hash"],
|
||||||
|
created_at=created_at,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.flush()
|
||||||
|
for item in parsed["items"]:
|
||||||
|
definition = self._resolve_definition(item, category)
|
||||||
|
self.session.add(
|
||||||
|
ProjectCriterionRecord(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=project_id,
|
||||||
|
version=version,
|
||||||
|
item_id=item["id"],
|
||||||
|
criterion_definition_id=definition.id,
|
||||||
|
criterion_key=definition.key,
|
||||||
|
operator=item["operator"],
|
||||||
|
value=item["value"],
|
||||||
|
enabled_for_discovery=item["enabled_for_discovery"],
|
||||||
|
created_at=created_at,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.flush()
|
||||||
|
|
||||||
|
def _resolve_definition(self, item: dict[str, Any], category: str | None):
|
||||||
|
if item.get("criterion_definition_id") is not None:
|
||||||
|
definition = get_definition(self.session, item["criterion_definition_id"])
|
||||||
|
if definition is None:
|
||||||
|
raise AppError(422, "UNKNOWN_CRITERION", "criterion definition not found")
|
||||||
|
return definition
|
||||||
|
existing = get_definition_by_key(self.session, str(item["criterion"]))
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
if not item.get("value_type"):
|
||||||
|
raise AppError(
|
||||||
|
422,
|
||||||
|
"UNKNOWN_CRITERION",
|
||||||
|
"criterion definition missing; provide value_type or create it first",
|
||||||
|
{"criterion": item["criterion"]},
|
||||||
|
)
|
||||||
|
return upsert_definition(
|
||||||
|
self.session,
|
||||||
|
{
|
||||||
|
"key": item["criterion"],
|
||||||
|
"origin": "INTERNAL",
|
||||||
|
"product_category_id": item.get("category") or category,
|
||||||
|
"label": item["criterion"],
|
||||||
|
"value_type": item["value_type"],
|
||||||
|
"unit": item.get("unit"),
|
||||||
|
"semantic": item.get("semantic"),
|
||||||
|
"category": item.get("category") or category,
|
||||||
|
"scoring_attribute_key": item["criterion"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _require_project(self, project_id: UUID) -> BuyingProjectRecord:
|
||||||
|
project = self.session.get(BuyingProjectRecord, project_id)
|
||||||
|
if project is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "buying project not found")
|
||||||
|
return project
|
||||||
|
|
||||||
|
def _out(self, project: BuyingProjectRecord, criteria: BuyingProjectCriteriaRecord | None) -> dict[str, Any]:
|
||||||
|
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
|
||||||
|
return jsonable(
|
||||||
|
{
|
||||||
|
"id": project.id,
|
||||||
|
"name": project.name,
|
||||||
|
"status": project.status,
|
||||||
|
"workspace_id": project.workspace_id,
|
||||||
|
"product_category_id": project.product_category_id,
|
||||||
|
"market": project.market,
|
||||||
|
"currency": project.currency,
|
||||||
|
"target_budget": project.target_budget,
|
||||||
|
"template_id": project.template_id,
|
||||||
|
"template_version": project.template_version,
|
||||||
|
"profile_id": project.template_id,
|
||||||
|
"profile_version": project.template_version,
|
||||||
|
"shortlist_size": project.shortlist_size,
|
||||||
|
"created_at": project.created_at,
|
||||||
|
"updated_at": project.updated_at,
|
||||||
|
"archived_at": project.archived_at,
|
||||||
|
"criteria_version": project.criteria_version if criteria is None else criteria.version,
|
||||||
|
"criteria": None if criteria is None else {"items": items},
|
||||||
|
"criteria_hash": None if criteria is None else criteria.definition_hash,
|
||||||
|
"ranking_status": RankingStatus.NOT_CONFIGURED.value
|
||||||
|
if project.template_id is None
|
||||||
|
else RankingStatus.CONFIGURED.value,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def _summary(self, project: BuyingProjectRecord) -> dict[str, Any]:
|
||||||
|
category = self.session.get(ProductCategoryRecord, project.product_category_id)
|
||||||
|
searches = list(
|
||||||
|
self.session.scalars(
|
||||||
|
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
active = next((row for row in searches if row.status not in INACTIVE_SEARCH), searches[0] if searches else None)
|
||||||
|
last_run = None
|
||||||
|
if active is not None:
|
||||||
|
last_run = self.session.scalars(
|
||||||
|
select(SearchRunRecord)
|
||||||
|
.where(SearchRunRecord.discovery_search_id == active.id)
|
||||||
|
.order_by(SearchRunRecord.started_at.desc())
|
||||||
|
).first()
|
||||||
|
candidates = list(
|
||||||
|
self.session.scalars(
|
||||||
|
select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project.id)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
counts = {"NEW": 0, "CHANGED": 0, "MISSING": 0}
|
||||||
|
if last_run is not None:
|
||||||
|
for item in self.session.scalars(
|
||||||
|
select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == last_run.id)
|
||||||
|
):
|
||||||
|
if item.diff_status in counts:
|
||||||
|
counts[item.diff_status] += 1
|
||||||
|
return jsonable(
|
||||||
|
{
|
||||||
|
"id": project.id,
|
||||||
|
"name": project.name,
|
||||||
|
"product_category": None
|
||||||
|
if category is None
|
||||||
|
else {"id": category.id, "name": category.name, "status": category.status},
|
||||||
|
"status": project.status,
|
||||||
|
"last_successful_run_at": None if active is None else active.last_successful_run_at,
|
||||||
|
"next_run_at": None if active is None else active.next_run_at,
|
||||||
|
"discovery_status": None if active is None else active.status,
|
||||||
|
"candidate_count": len(candidates),
|
||||||
|
"shortlist_count": sum(1 for row in candidates if row.status == CandidateStatus.SHORTLISTED.value),
|
||||||
|
"new_count": last_run.new_count if last_run is not None else counts[DiffStatus.NEW.value],
|
||||||
|
"changed_count": last_run.changed_count if last_run is not None else counts[DiffStatus.CHANGED.value],
|
||||||
|
"missing_count": last_run.missing_count if last_run is not None else counts[DiffStatus.MISSING.value],
|
||||||
|
"current_search_result_count": None if last_run is None else last_run.result_count,
|
||||||
|
"ranking_status": RankingStatus.NOT_CONFIGURED.value
|
||||||
|
if project.template_id is None
|
||||||
|
else RankingStatus.CONFIGURED.value,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from datetime import datetime
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
@ -13,6 +14,12 @@ class Settings(BaseSettings):
|
||||||
db_password: str = "change_me"
|
db_password: str = "change_me"
|
||||||
environment: str = "development"
|
environment: str = "development"
|
||||||
engine_build: str = "dev"
|
engine_build: str = "dev"
|
||||||
|
clock_now: datetime | None = None
|
||||||
|
cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cors_origins(self) -> list[str]:
|
||||||
|
return [item.strip() for item in self.cors_allowed_origins.split(",") if item.strip()]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def database_url(self) -> str:
|
def database_url(self) -> str:
|
||||||
0
backend/app/sources/__init__.py
Normal file
0
backend/app/sources/__init__.py
Normal file
85
backend/app/sources/base.py
Normal file
85
backend/app/sources/base.py
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any, Protocol
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SourceProduct:
|
||||||
|
source_key: str
|
||||||
|
external_id: str
|
||||||
|
id: UUID
|
||||||
|
category: str
|
||||||
|
brand: str
|
||||||
|
name: str
|
||||||
|
model: str | None
|
||||||
|
variants: tuple["SourceVariant", ...]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SourceVariant:
|
||||||
|
id: UUID
|
||||||
|
name: str
|
||||||
|
manufacturer_sku: str | None
|
||||||
|
identifiers: dict[str, Any]
|
||||||
|
attributes: tuple["SourceAttribute", ...]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SourceAttribute:
|
||||||
|
id: UUID
|
||||||
|
attribute: str
|
||||||
|
raw_value: Any
|
||||||
|
normalized_value: Any
|
||||||
|
unit: str | None
|
||||||
|
normalization_status: str
|
||||||
|
normalizer_version: str
|
||||||
|
confidence: Decimal
|
||||||
|
source_reference: str
|
||||||
|
observed_at: datetime
|
||||||
|
normalization_error: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SourceOffer:
|
||||||
|
id: UUID
|
||||||
|
variant_id: UUID
|
||||||
|
external_id: str
|
||||||
|
seller: str
|
||||||
|
url: str | None
|
||||||
|
market: str
|
||||||
|
currency: str
|
||||||
|
condition: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SourcePriceObservation:
|
||||||
|
id: UUID | None
|
||||||
|
external_offer_id: str
|
||||||
|
external_observation_id: str
|
||||||
|
observed_at: datetime
|
||||||
|
item_price: Decimal
|
||||||
|
shipping_price: Decimal | None
|
||||||
|
currency: str
|
||||||
|
availability: str
|
||||||
|
raw_payload: Any
|
||||||
|
confidence: Decimal
|
||||||
|
|
||||||
|
|
||||||
|
class ProductSource(Protocol):
|
||||||
|
source_key: str
|
||||||
|
|
||||||
|
def get_product(self, external_id: str) -> SourceProduct: ...
|
||||||
|
def get_offers(self, external_product_id: str) -> list[SourceOffer]: ...
|
||||||
|
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]: ...
|
||||||
|
|
||||||
|
|
||||||
|
class DiscoverySourceAdapter(Protocol):
|
||||||
|
source_key: str
|
||||||
|
|
||||||
|
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]: ...
|
||||||
|
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: ...
|
||||||
|
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]: ...
|
||||||
|
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]: ...
|
||||||
|
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]: ...
|
||||||
3
backend/app/sources/geizhals/__init__.py
Normal file
3
backend/app/sources/geizhals/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
from app.sources.geizhals.adapter import GeizhalsAdapter
|
||||||
|
|
||||||
|
__all__ = ["GeizhalsAdapter"]
|
||||||
81
backend/app/sources/geizhals/adapter.py
Normal file
81
backend/app/sources/geizhals/adapter.py
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import parse_qs, urlparse
|
||||||
|
|
||||||
|
from app.discovery.enums import FilterType
|
||||||
|
from app.errors import AppError
|
||||||
|
|
||||||
|
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
||||||
|
|
||||||
|
|
||||||
|
class GeizhalsAdapter:
|
||||||
|
source_key = "geizhals"
|
||||||
|
|
||||||
|
def __init__(self, fixture_dir: Path | None = None, live: bool = False) -> None:
|
||||||
|
self.fixture_dir = fixture_dir or FIXTURE_DIR
|
||||||
|
self.live = live
|
||||||
|
|
||||||
|
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]:
|
||||||
|
payload = self._load(f"{category_ref}-taxonomy.json")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||||
|
from app.discovery.urlgen import generate_search_url
|
||||||
|
|
||||||
|
category_url = f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
|
||||||
|
url, internal_only = generate_search_url(
|
||||||
|
category_url=category_url,
|
||||||
|
criteria_items=criteria,
|
||||||
|
mappings=mappings,
|
||||||
|
)
|
||||||
|
return {"url": url, "internal_only": internal_only}
|
||||||
|
|
||||||
|
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
if self.live:
|
||||||
|
raise AppError(422, "LIVE_FETCH_DISABLED", "live Geizhals fetch is not enabled in this process")
|
||||||
|
page = int(pagination.get("page") or 1)
|
||||||
|
category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0]
|
||||||
|
return self._load(f"{category}-search-page{page}.json")
|
||||||
|
|
||||||
|
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
return list(response.get("items") or [])
|
||||||
|
|
||||||
|
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
attrs = result.get("normalized_attributes") or {}
|
||||||
|
return {
|
||||||
|
"source_external_id": result.get("source_external_id"),
|
||||||
|
"source_url": result.get("source_url"),
|
||||||
|
"source_title": result.get("source_title"),
|
||||||
|
"manufacturer": result.get("manufacturer"),
|
||||||
|
"model": result.get("model"),
|
||||||
|
"manufacturer_sku": result.get("manufacturer_sku"),
|
||||||
|
"gtin_ean": result.get("gtin_ean"),
|
||||||
|
"best_price": result.get("best_price"),
|
||||||
|
"currency": result.get("currency") or "EUR",
|
||||||
|
"availability": result.get("availability") or "unknown",
|
||||||
|
"seller_count": result.get("seller_count"),
|
||||||
|
"raw_attributes": result.get("raw_attributes") or {},
|
||||||
|
"normalized_attributes": attrs,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _load(self, name: str) -> dict[str, Any]:
|
||||||
|
path = self.fixture_dir / name
|
||||||
|
if not path.exists():
|
||||||
|
raise AppError(404, "FIXTURE_NOT_FOUND", f"geizhals fixture missing: {name}")
|
||||||
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
def _category(self, mappings: list[dict[str, Any]], criteria: list[dict[str, Any]]) -> str:
|
||||||
|
for mapping in mappings:
|
||||||
|
if mapping.get("category_key"):
|
||||||
|
return str(mapping["category_key"])
|
||||||
|
return "nb"
|
||||||
|
|
||||||
|
|
||||||
|
def filter_type(raw: str) -> str:
|
||||||
|
try:
|
||||||
|
return FilterType(raw).value
|
||||||
|
except ValueError:
|
||||||
|
return FilterType.UNKNOWN.value
|
||||||
110
backend/app/sources/manual.py
Normal file
110
backend/app/sources/manual.py
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
import json
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.sources.base import (
|
||||||
|
SourceAttribute,
|
||||||
|
SourceOffer,
|
||||||
|
SourcePriceObservation,
|
||||||
|
SourceProduct,
|
||||||
|
SourceVariant,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_dt(value: str) -> datetime:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
|
||||||
|
|
||||||
|
class ManualSource:
|
||||||
|
source_key = "manual"
|
||||||
|
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = path
|
||||||
|
self._data = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
self._products = {item["external_id"]: item for item in self._data["products"]}
|
||||||
|
self._offers = {item["external_id"]: item for item in self._data.get("offers", [])}
|
||||||
|
self._prices = self._data.get("prices", [])
|
||||||
|
|
||||||
|
def list_external_product_ids(self) -> list[str]:
|
||||||
|
return list(self._products)
|
||||||
|
|
||||||
|
def get_product(self, external_id: str) -> SourceProduct:
|
||||||
|
raw = self._products.get(external_id)
|
||||||
|
if raw is None:
|
||||||
|
raise AppError(404, "NOT_FOUND", "manual product not found", {"external_id": external_id})
|
||||||
|
variants = []
|
||||||
|
for variant in raw["variants"]:
|
||||||
|
attributes = tuple(
|
||||||
|
SourceAttribute(
|
||||||
|
id=UUID(item["id"]),
|
||||||
|
attribute=item["attribute"],
|
||||||
|
raw_value=item["raw_value"],
|
||||||
|
normalized_value=item.get("normalized_value"),
|
||||||
|
unit=item.get("unit"),
|
||||||
|
normalization_status=item["normalization_status"],
|
||||||
|
normalizer_version=item.get("normalizer_version", "manual-v1"),
|
||||||
|
confidence=Decimal(str(item["confidence"])),
|
||||||
|
source_reference=item["source_reference"],
|
||||||
|
observed_at=_parse_dt(item["observed_at"]),
|
||||||
|
normalization_error=item.get("normalization_error"),
|
||||||
|
)
|
||||||
|
for item in variant["attributes"]
|
||||||
|
)
|
||||||
|
variants.append(
|
||||||
|
SourceVariant(
|
||||||
|
id=UUID(variant["id"]),
|
||||||
|
name=variant["name"],
|
||||||
|
manufacturer_sku=variant.get("manufacturer_sku"),
|
||||||
|
identifiers=variant.get("identifiers") or {},
|
||||||
|
attributes=attributes,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return SourceProduct(
|
||||||
|
source_key=self.source_key,
|
||||||
|
external_id=external_id,
|
||||||
|
id=UUID(raw["id"]),
|
||||||
|
category=raw["category"],
|
||||||
|
brand=raw["brand"],
|
||||||
|
name=raw["name"],
|
||||||
|
model=raw.get("model"),
|
||||||
|
variants=tuple(variants),
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_offers(self, external_product_id: str) -> list[SourceOffer]:
|
||||||
|
product = self.get_product(external_product_id)
|
||||||
|
variant_ids = {variant.id for variant in product.variants}
|
||||||
|
return [
|
||||||
|
SourceOffer(
|
||||||
|
id=UUID(item["id"]),
|
||||||
|
variant_id=UUID(item["variant_id"]),
|
||||||
|
external_id=item["external_id"],
|
||||||
|
seller=item["seller"],
|
||||||
|
url=item.get("url"),
|
||||||
|
market=item["market"],
|
||||||
|
currency=item["currency"],
|
||||||
|
condition=item["condition"],
|
||||||
|
)
|
||||||
|
for item in self._offers.values()
|
||||||
|
if UUID(item["variant_id"]) in variant_ids
|
||||||
|
]
|
||||||
|
|
||||||
|
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]:
|
||||||
|
return [
|
||||||
|
SourcePriceObservation(
|
||||||
|
id=UUID(item["id"]) if item.get("id") else None,
|
||||||
|
external_offer_id=external_offer_id,
|
||||||
|
external_observation_id=item["external_observation_id"],
|
||||||
|
observed_at=_parse_dt(item["observed_at"]),
|
||||||
|
item_price=Decimal(str(item["item_price"])),
|
||||||
|
shipping_price=None if item.get("shipping_price") is None else Decimal(str(item["shipping_price"])),
|
||||||
|
currency=item["currency"],
|
||||||
|
availability=item["availability"],
|
||||||
|
raw_payload=item["raw_payload"],
|
||||||
|
confidence=Decimal(str(item["confidence"])),
|
||||||
|
)
|
||||||
|
for item in self._prices
|
||||||
|
if item["external_offer_id"] == external_offer_id
|
||||||
|
]
|
||||||
38
backend/fixtures/geizhals/nb-search-page1.json
Normal file
38
backend/fixtures/geizhals/nb-search-page1.json
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
{
|
||||||
|
"page": 1,
|
||||||
|
"page_count": 2,
|
||||||
|
"result_count": 3,
|
||||||
|
"http_status": 200,
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"source_external_id": "gh-nb-a",
|
||||||
|
"source_url": "https://geizhals.de/example-mobile-13-a.html",
|
||||||
|
"source_title": "Example Mobile 13 32GB",
|
||||||
|
"manufacturer": "Example",
|
||||||
|
"model": "Mobile 13",
|
||||||
|
"manufacturer_sku": "EM13-32-1T",
|
||||||
|
"gtin_ean": "4000000000001",
|
||||||
|
"best_price": "1298.00",
|
||||||
|
"currency": "EUR",
|
||||||
|
"availability": "in_stock",
|
||||||
|
"seller_count": 8,
|
||||||
|
"raw_attributes": {"RAM": "32GB"},
|
||||||
|
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_external_id": "gh-nb-b",
|
||||||
|
"source_url": "https://geizhals.de/example-mobile-13-b.html",
|
||||||
|
"source_title": "Example Mobile 13 8GB",
|
||||||
|
"manufacturer": "Example",
|
||||||
|
"model": "Mobile 13",
|
||||||
|
"manufacturer_sku": "EM13-8-1T",
|
||||||
|
"gtin_ean": "4000000000002",
|
||||||
|
"best_price": "1298.00",
|
||||||
|
"currency": "EUR",
|
||||||
|
"availability": "in_stock",
|
||||||
|
"seller_count": 5,
|
||||||
|
"raw_attributes": {"RAM": "8GB"},
|
||||||
|
"normalized_attributes": {"ram_gb": 8, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
23
backend/fixtures/geizhals/nb-search-page2.json
Normal file
23
backend/fixtures/geizhals/nb-search-page2.json
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
{
|
||||||
|
"page": 2,
|
||||||
|
"page_count": 2,
|
||||||
|
"result_count": 3,
|
||||||
|
"http_status": 200,
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"source_external_id": "gh-nb-c",
|
||||||
|
"source_url": "https://geizhals.de/example-mobile-13-c.html",
|
||||||
|
"source_title": "Example Mobile 13 unknown weight",
|
||||||
|
"manufacturer": "Example",
|
||||||
|
"model": "Mobile 13 UW",
|
||||||
|
"manufacturer_sku": "EM13-32-UW",
|
||||||
|
"gtin_ean": "4000000000003",
|
||||||
|
"best_price": "1298.00",
|
||||||
|
"currency": "EUR",
|
||||||
|
"availability": "in_stock",
|
||||||
|
"seller_count": 3,
|
||||||
|
"raw_attributes": {"RAM": "32GB"},
|
||||||
|
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
27
backend/fixtures/geizhals/nb-taxonomy.json
Normal file
27
backend/fixtures/geizhals/nb-taxonomy.json
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
{
|
||||||
|
"source": "geizhals",
|
||||||
|
"external_category_key": "nb",
|
||||||
|
"name": "Notebooks",
|
||||||
|
"canonical_category": "notebook",
|
||||||
|
"source_url": "https://geizhals.de/?cat=nb",
|
||||||
|
"filters": [
|
||||||
|
{
|
||||||
|
"external_filter_key": "17023",
|
||||||
|
"label": "Arbeitsspeicher",
|
||||||
|
"filter_type": "NUMERIC_THRESHOLD",
|
||||||
|
"canonical_unit": "GB",
|
||||||
|
"options": [
|
||||||
|
{"source_value": "ab 16GB", "normalized_value": 16, "operator": ">=", "unit": "GB"},
|
||||||
|
{"source_value": "ab 24GB", "normalized_value": 24, "operator": ">=", "unit": "GB"},
|
||||||
|
{"source_value": "ab 32GB", "normalized_value": 32, "operator": ">=", "unit": "GB"}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_filter_key": "price",
|
||||||
|
"label": "Preis",
|
||||||
|
"filter_type": "NUMERIC_RANGE",
|
||||||
|
"canonical_unit": "EUR",
|
||||||
|
"range": {"minimum": 0, "maximum": null, "step": 1, "unit": "EUR"}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
80
backend/fixtures/manual-demo.json
Normal file
80
backend/fixtures/manual-demo.json
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
{
|
||||||
|
"products": [
|
||||||
|
{
|
||||||
|
"id": "11111111-1111-4111-8111-111111111111",
|
||||||
|
"external_id": "example-mobile-13",
|
||||||
|
"category": "notebook",
|
||||||
|
"brand": "Example",
|
||||||
|
"name": "Example Mobile 13",
|
||||||
|
"model": "EM13",
|
||||||
|
"variants": [
|
||||||
|
{
|
||||||
|
"id": "22222222-2222-4222-8222-222222222222",
|
||||||
|
"name": "32 GB / 1 TB",
|
||||||
|
"manufacturer_sku": "EM13-32-1T",
|
||||||
|
"identifiers": {"sku": "EM13-32-1T"},
|
||||||
|
"attributes": [
|
||||||
|
{"id": "aaaaaaa1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "aaaaaaa1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "aaaaaaa1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/weight", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "aaaaaaa1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "aaaaaaa1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/display", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "aaaaaaa1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "22222222-2222-4222-8222-22222222222b",
|
||||||
|
"name": "8 GB / 1 TB",
|
||||||
|
"manufacturer_sku": "EM13-8-1T",
|
||||||
|
"identifiers": {"sku": "EM13-8-1T"},
|
||||||
|
"attributes": [
|
||||||
|
{"id": "bbbbbbb1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "8GB RAM", "normalized_value": 8, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "bbbbbbb1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "bbbbbbb1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/weight", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "bbbbbbb1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "bbbbbbb1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/display", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "bbbbbbb1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "22222222-2222-4222-8222-22222222222c",
|
||||||
|
"name": "32 GB / unknown weight",
|
||||||
|
"manufacturer_sku": "EM13-32-UW",
|
||||||
|
"identifiers": {},
|
||||||
|
"attributes": [
|
||||||
|
{"id": "ccccccc1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "ccccccc1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "ccccccc1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "weight unknown", "normalized_value": null, "unit": "kg", "normalization_status": "unknown", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/weight", "observed_at": "2026-10-03T09:00:00Z", "normalization_error": "manual unknown"},
|
||||||
|
{"id": "ccccccc1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "ccccccc1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/display", "observed_at": "2026-10-03T09:00:00Z"},
|
||||||
|
{"id": "ccccccc1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"offers": [
|
||||||
|
{"id": "33333333-3333-4333-8333-333333333333", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-a", "seller": "DemoShop", "url": "https://example.test/a", "market": "DE", "currency": "EUR", "condition": "new"},
|
||||||
|
{"id": "33333333-3333-4333-8333-33333333333b", "variant_id": "22222222-2222-4222-8222-22222222222b", "external_id": "demo-offer-b", "seller": "DemoShop", "url": "https://example.test/b", "market": "DE", "currency": "EUR", "condition": "new"},
|
||||||
|
{"id": "33333333-3333-4333-8333-33333333333c", "variant_id": "22222222-2222-4222-8222-22222222222c", "external_id": "demo-offer-c", "seller": "DemoShop", "url": "https://example.test/c", "market": "DE", "currency": "EUR", "condition": "new"},
|
||||||
|
{"id": "33333333-3333-4333-8333-33333333333d", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-d", "seller": "OtherShop", "url": "https://example.test/d", "market": "DE", "currency": "EUR", "condition": "new"}
|
||||||
|
],
|
||||||
|
"prices": [
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000a1", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000a2", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000a3", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000a4", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000b1", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000b2", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000b3", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000b4", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000c1", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000c2", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000c3", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000c4", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000d1", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000d2", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000d3", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||||
|
{"id": "ddddddd1-0000-4000-8000-0000000000d4", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "2049.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "2049.00"}}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
"""Deploy skeleton — domain schema follows in a later revision.
|
"""Deploy skeleton — domain schema follows in 0002.
|
||||||
|
|
||||||
Revision ID: 0001_deploy_skeleton
|
Revision ID: 0001_deploy_skeleton
|
||||||
Revises:
|
Revises:
|
||||||
233
backend/migrations/versions/0002_domain_schema.py
Normal file
233
backend/migrations/versions/0002_domain_schema.py
Normal file
|
|
@ -0,0 +1,233 @@
|
||||||
|
"""Global product/price schema for v0.1. No user or workspace columns.
|
||||||
|
|
||||||
|
Revision ID: 0002_domain_schema
|
||||||
|
Revises: 0001_deploy_skeleton
|
||||||
|
Create Date: 2026-10-03
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0002_domain_schema"
|
||||||
|
down_revision: str | None = "0001_deploy_skeleton"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"attribute_definitions",
|
||||||
|
sa.Column("key", sa.String(128), primary_key=True),
|
||||||
|
sa.Column("value_type", sa.String(32), nullable=False),
|
||||||
|
sa.Column("canonical_unit", sa.String(32)),
|
||||||
|
sa.Column("minimum", sa.Numeric(12, 4)),
|
||||||
|
sa.Column("maximum", sa.Numeric(12, 4)),
|
||||||
|
sa.Column("description", sa.Text()),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"value_type IN ('number', 'string', 'boolean', 'string_list')",
|
||||||
|
name="ck_attribute_definitions_value_type",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"sources",
|
||||||
|
sa.Column("id", sa.String(64), primary_key=True),
|
||||||
|
sa.Column("name", sa.String(128), nullable=False),
|
||||||
|
sa.Column("adapter_type", sa.String(64), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"scoring_models",
|
||||||
|
sa.Column("version", sa.String(64), primary_key=True),
|
||||||
|
sa.Column("algorithm_id", sa.String(128), nullable=False),
|
||||||
|
sa.Column("definition", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"product_profiles",
|
||||||
|
sa.Column("id", sa.String(128), primary_key=True),
|
||||||
|
sa.Column("version", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("name", sa.String(256), nullable=False),
|
||||||
|
sa.Column("category", sa.String(128), nullable=False),
|
||||||
|
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
|
||||||
|
sa.Column("definition", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.CheckConstraint("version > 0", name="ck_product_profiles_version"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"products",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("category", sa.String(128), nullable=False),
|
||||||
|
sa.Column("brand", sa.String(128), nullable=False),
|
||||||
|
sa.Column("name", sa.String(256), nullable=False),
|
||||||
|
sa.Column("model", sa.String(128)),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"product_variants",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("product_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("products.id"), nullable=False),
|
||||||
|
sa.Column("name", sa.String(256), nullable=False),
|
||||||
|
sa.Column("manufacturer_sku", sa.String(128)),
|
||||||
|
sa.Column("identifiers", postgresql.JSONB, nullable=False, server_default=sa.text("'{}'::jsonb")),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_index("ix_product_variants_product_id", "product_variants", ["product_id"])
|
||||||
|
op.create_table(
|
||||||
|
"variant_attributes",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
||||||
|
sa.Column("attribute_key", sa.String(128), sa.ForeignKey("attribute_definitions.key"), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("raw_value", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("normalized_value", postgresql.JSONB),
|
||||||
|
sa.Column("unit", sa.String(32)),
|
||||||
|
sa.Column("normalization_status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("normalizer_version", sa.String(64), nullable=False),
|
||||||
|
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
|
||||||
|
sa.Column("source_reference", sa.String(256), nullable=False),
|
||||||
|
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("normalization_error", sa.Text()),
|
||||||
|
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"normalization_status IN ('normalized', 'unknown')",
|
||||||
|
name="ck_variant_attributes_status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX ix_variant_attributes_lookup ON variant_attributes "
|
||||||
|
"(variant_id, attribute_key, observed_at DESC, ingested_at DESC, id DESC)"
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"offers",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("external_id", sa.String(256), nullable=False),
|
||||||
|
sa.Column("seller", sa.String(256), nullable=False),
|
||||||
|
sa.Column("url", sa.Text()),
|
||||||
|
sa.Column("market", sa.String(16), nullable=False),
|
||||||
|
sa.Column("currency", sa.String(3), nullable=False),
|
||||||
|
sa.Column("condition", sa.String(32), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
|
||||||
|
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
|
||||||
|
sa.CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_offers_variant_id", "offers", ["variant_id"])
|
||||||
|
op.create_table(
|
||||||
|
"price_observations",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("external_observation_id", sa.String(256), nullable=False),
|
||||||
|
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("item_price", sa.Numeric(12, 2), nullable=False),
|
||||||
|
sa.Column("shipping_price", sa.Numeric(12, 2)),
|
||||||
|
sa.Column("currency", sa.String(3), nullable=False),
|
||||||
|
sa.Column("availability", sa.String(32), nullable=False),
|
||||||
|
sa.Column("raw_payload", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
|
||||||
|
sa.UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
|
||||||
|
sa.CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
|
||||||
|
sa.CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
|
||||||
|
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
|
||||||
|
name="ck_price_obs_availability",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX ix_price_observations_lookup ON price_observations "
|
||||||
|
"(offer_id, observed_at DESC, ingested_at DESC, id DESC)"
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"evaluations",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
||||||
|
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
|
||||||
|
sa.Column("profile_id", sa.String(128), nullable=False),
|
||||||
|
sa.Column("profile_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
|
||||||
|
sa.Column("as_of", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("snapshot_cutoff", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("scoring_hash", sa.String(64), nullable=False),
|
||||||
|
sa.Column("engine_build", sa.String(64), nullable=False),
|
||||||
|
sa.Column("product_eligibility", sa.String(16), nullable=False),
|
||||||
|
sa.Column("offer_eligibility", sa.String(16), nullable=False),
|
||||||
|
sa.Column("overall_eligibility", sa.String(16), nullable=False),
|
||||||
|
sa.Column("product_score", sa.Numeric(6, 2)),
|
||||||
|
sa.Column("price_score", sa.Numeric(6, 2)),
|
||||||
|
sa.Column("deal_score", sa.Numeric(6, 2)),
|
||||||
|
sa.Column("classification", sa.String(32), nullable=False),
|
||||||
|
sa.Column("price_reference_scope", sa.String(32), nullable=False),
|
||||||
|
sa.Column("input_snapshot", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("result_snapshot", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["profile_id", "profile_version"],
|
||||||
|
["product_profiles.id", "product_profiles.version"],
|
||||||
|
name="fk_evaluations_profile",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||||
|
name="ck_eval_product_eligibility",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||||
|
name="ck_eval_offer_eligibility",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||||
|
name="ck_eval_overall_eligibility",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
|
||||||
|
name="ck_eval_product_score",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
|
||||||
|
name="ck_eval_price_score",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
|
||||||
|
name="ck_eval_deal_score",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_evaluations_variant_id", "evaluations", ["variant_id"])
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX ix_evaluations_profile_created ON evaluations "
|
||||||
|
"(profile_id, profile_version, created_at DESC)"
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"evaluation_components",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("evaluation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("evaluations.id"), nullable=False),
|
||||||
|
sa.Column("kind", sa.String(64), nullable=False),
|
||||||
|
sa.Column("component_key", sa.String(128), nullable=False),
|
||||||
|
sa.Column("raw_score", sa.Numeric(12, 6)),
|
||||||
|
sa.Column("weight", sa.Numeric(12, 6)),
|
||||||
|
sa.Column("contribution", sa.Numeric(12, 6)),
|
||||||
|
sa.Column("details", postgresql.JSONB, nullable=False),
|
||||||
|
sa.UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("evaluation_components")
|
||||||
|
op.drop_table("evaluations")
|
||||||
|
op.drop_table("price_observations")
|
||||||
|
op.drop_table("offers")
|
||||||
|
op.drop_table("variant_attributes")
|
||||||
|
op.drop_table("product_variants")
|
||||||
|
op.drop_table("products")
|
||||||
|
op.drop_table("product_profiles")
|
||||||
|
op.drop_table("scoring_models")
|
||||||
|
op.drop_table("sources")
|
||||||
|
op.drop_table("attribute_definitions")
|
||||||
229
backend/migrations/versions/0003_discovery_schema.py
Normal file
229
backend/migrations/versions/0003_discovery_schema.py
Normal file
|
|
@ -0,0 +1,229 @@
|
||||||
|
"""Discovery, taxonomy, search runs, and project candidates.
|
||||||
|
|
||||||
|
Revision ID: 0003_discovery_schema
|
||||||
|
Revises: 0002_domain_schema
|
||||||
|
Create Date: 2026-10-04
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0003_discovery_schema"
|
||||||
|
down_revision: str | None = "0002_domain_schema"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"buying_projects",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("name", sa.String(256), nullable=False),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("workspace_id", sa.String(64)),
|
||||||
|
sa.Column("profile_id", sa.String(128), nullable=False),
|
||||||
|
sa.Column("profile_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("shortlist_size", sa.Integer(), nullable=False, server_default="5"),
|
||||||
|
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("archived_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["profile_id", "profile_version"],
|
||||||
|
["product_profiles.id", "product_profiles.version"],
|
||||||
|
name="fk_buying_projects_profile",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("status IN ('ACTIVE', 'PURCHASED', 'ARCHIVED')", name="ck_buying_projects_status"),
|
||||||
|
sa.CheckConstraint("shortlist_size > 0", name="ck_buying_projects_shortlist"),
|
||||||
|
sa.CheckConstraint("schedule_interval_minutes > 0", name="ck_buying_projects_interval"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"buying_project_criteria",
|
||||||
|
sa.Column("project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), primary_key=True),
|
||||||
|
sa.Column("version", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("definition", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.CheckConstraint("version > 0", name="ck_project_criteria_version"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"source_categories",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("external_category_key", sa.String(64), nullable=False),
|
||||||
|
sa.Column("name", sa.String(256), nullable=False),
|
||||||
|
sa.Column("source_url", sa.Text(), nullable=False),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("last_verified_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"source_filter_definitions",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||||
|
sa.Column("external_filter_key", sa.String(64), nullable=False),
|
||||||
|
sa.Column("label", sa.String(256), nullable=False),
|
||||||
|
sa.Column("filter_type", sa.String(32), nullable=False),
|
||||||
|
sa.Column("canonical_unit", sa.String(32)),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"source_filter_options",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), nullable=False),
|
||||||
|
sa.Column("source_value", sa.String(256), nullable=False),
|
||||||
|
sa.Column("normalized_value", postgresql.JSONB),
|
||||||
|
sa.Column("operator", sa.String(16)),
|
||||||
|
sa.Column("unit", sa.String(32)),
|
||||||
|
sa.UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"source_filter_ranges",
|
||||||
|
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), primary_key=True),
|
||||||
|
sa.Column("minimum", sa.Numeric(12, 2)),
|
||||||
|
sa.Column("maximum", sa.Numeric(12, 2)),
|
||||||
|
sa.Column("step", sa.Numeric(12, 2)),
|
||||||
|
sa.Column("unit", sa.String(32)),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"criterion_definitions",
|
||||||
|
sa.Column("key", sa.String(128), primary_key=True),
|
||||||
|
sa.Column("value_type", sa.String(32), nullable=False),
|
||||||
|
sa.Column("unit", sa.String(32)),
|
||||||
|
sa.Column("semantic", sa.String(128)),
|
||||||
|
sa.Column("description", sa.Text()),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"criterion_mappings",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||||
|
sa.Column("external_filter_key", sa.String(64), nullable=False),
|
||||||
|
sa.Column("source_value", sa.String(256)),
|
||||||
|
sa.Column("encoding", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"discovery_searches",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||||
|
sa.Column("criteria_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("mapping_version", sa.String(64), nullable=False),
|
||||||
|
sa.Column("generated_url", sa.Text(), nullable=False),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("broad_result_threshold", sa.Integer(), nullable=False, server_default="300"),
|
||||||
|
sa.Column("next_run_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("last_successful_run_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"search_runs",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("discovery_search_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("discovery_searches.id"), nullable=False),
|
||||||
|
sa.Column("criteria_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("mapping_version", sa.String(64), nullable=False),
|
||||||
|
sa.Column("generated_url", sa.Text(), nullable=False),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("result_count", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("http_status_summary", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("error_code", sa.String(64)),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"source_product_references",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||||
|
sa.Column("source_external_id", sa.String(256)),
|
||||||
|
sa.Column("source_url", sa.Text()),
|
||||||
|
sa.Column("source_title", sa.String(512), nullable=False),
|
||||||
|
sa.Column("manufacturer", sa.String(256)),
|
||||||
|
sa.Column("model", sa.String(256)),
|
||||||
|
sa.Column("manufacturer_sku", sa.String(128)),
|
||||||
|
sa.Column("gtin_ean", sa.String(32)),
|
||||||
|
sa.Column("normalized_spec_fingerprint", sa.Text()),
|
||||||
|
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("lifecycle_status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("match_status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("consecutive_missing_success_runs", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"product_observations",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
||||||
|
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
|
||||||
|
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("raw_title", sa.String(512), nullable=False),
|
||||||
|
sa.Column("best_price", sa.Numeric(12, 2)),
|
||||||
|
sa.Column("currency", sa.String(3)),
|
||||||
|
sa.Column("raw_attributes", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False),
|
||||||
|
sa.Column("availability", sa.String(32)),
|
||||||
|
sa.Column("seller_count", sa.Integer()),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"search_run_items",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
|
||||||
|
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
||||||
|
sa.Column("observation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_observations.id")),
|
||||||
|
sa.Column("diff_status", sa.String(32), nullable=False),
|
||||||
|
sa.UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"project_candidates",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
|
||||||
|
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
||||||
|
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("match_status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("product_eligibility", sa.String(16)),
|
||||||
|
sa.Column("product_score", sa.Numeric(6, 2)),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"candidate_status_history",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("candidate_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("project_candidates.id"), nullable=False),
|
||||||
|
sa.Column("from_status", sa.String(32)),
|
||||||
|
sa.Column("to_status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("reason", sa.String(128)),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_index("ix_discovery_searches_next_run", "discovery_searches", ["next_run_at"])
|
||||||
|
op.create_index("ix_search_runs_search", "search_runs", ["discovery_search_id", "started_at"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("candidate_status_history")
|
||||||
|
op.drop_table("project_candidates")
|
||||||
|
op.drop_table("search_run_items")
|
||||||
|
op.drop_table("product_observations")
|
||||||
|
op.drop_table("source_product_references")
|
||||||
|
op.drop_table("search_runs")
|
||||||
|
op.drop_table("discovery_searches")
|
||||||
|
op.drop_table("criterion_mappings")
|
||||||
|
op.drop_table("criterion_definitions")
|
||||||
|
op.drop_table("source_filter_ranges")
|
||||||
|
op.drop_table("source_filter_options")
|
||||||
|
op.drop_table("source_filter_definitions")
|
||||||
|
op.drop_table("source_categories")
|
||||||
|
op.drop_table("buying_project_criteria")
|
||||||
|
op.drop_table("buying_projects")
|
||||||
127
backend/migrations/versions/0004_dynamic_attributes_criteria.py
Normal file
127
backend/migrations/versions/0004_dynamic_attributes_criteria.py
Normal file
|
|
@ -0,0 +1,127 @@
|
||||||
|
"""Dynamic attribute and criterion value records.
|
||||||
|
|
||||||
|
Revision ID: 0004_dynamic_attributes
|
||||||
|
Revises: 0003_discovery_schema
|
||||||
|
Create Date: 2026-10-04
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0004_dynamic_attributes"
|
||||||
|
down_revision: str | None = "0003_discovery_schema"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("criterion_definitions", sa.Column("category", sa.String(128)))
|
||||||
|
op.add_column("source_categories", sa.Column("canonical_category", sa.String(128)))
|
||||||
|
op.create_table(
|
||||||
|
"project_criteria",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||||
|
sa.Column("version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("item_id", sa.String(128), nullable=False),
|
||||||
|
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
|
||||||
|
sa.Column("operator", sa.String(16), nullable=False),
|
||||||
|
sa.Column("value", postgresql.JSONB),
|
||||||
|
sa.Column("mandatory", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["project_id", "version"],
|
||||||
|
["buying_project_criteria.project_id", "buying_project_criteria.version"],
|
||||||
|
name="fk_project_criteria_version",
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"product_observation_attributes",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"observation_id",
|
||||||
|
postgresql.UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("product_observations.id"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("attribute_key", sa.String(128), nullable=False),
|
||||||
|
sa.Column("raw_value", postgresql.JSONB),
|
||||||
|
sa.Column("normalized_value", postgresql.JSONB),
|
||||||
|
sa.Column("unit", sa.String(32)),
|
||||||
|
sa.UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),
|
||||||
|
)
|
||||||
|
_migrate_criteria_json()
|
||||||
|
_migrate_observation_json()
|
||||||
|
op.drop_column("buying_project_criteria", "definition")
|
||||||
|
op.drop_column("product_observations", "raw_attributes")
|
||||||
|
op.drop_column("product_observations", "normalized_attributes")
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_criteria_json() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
rows = bind.execute(sa.text("SELECT project_id, version, definition FROM buying_project_criteria")).mappings()
|
||||||
|
for row in rows:
|
||||||
|
items = (row["definition"] or {}).get("items") or []
|
||||||
|
for raw in items:
|
||||||
|
bind.execute(
|
||||||
|
sa.text(
|
||||||
|
"""
|
||||||
|
INSERT INTO project_criteria
|
||||||
|
(id, project_id, version, item_id, criterion_key, operator, value, mandatory)
|
||||||
|
VALUES
|
||||||
|
(gen_random_uuid(), :project_id, :version, :item_id, :criterion_key, :operator, :value, :mandatory)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"project_id": row["project_id"],
|
||||||
|
"version": row["version"],
|
||||||
|
"item_id": str(raw.get("id") or raw.get("criterion")),
|
||||||
|
"criterion_key": str(raw.get("criterion")),
|
||||||
|
"operator": str(raw.get("operator")),
|
||||||
|
"value": raw.get("value"),
|
||||||
|
"mandatory": bool(raw.get("mandatory", True)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_observation_json() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
rows = bind.execute(
|
||||||
|
sa.text("SELECT id, raw_attributes, normalized_attributes FROM product_observations")
|
||||||
|
).mappings()
|
||||||
|
for row in rows:
|
||||||
|
raw = row["raw_attributes"] or {}
|
||||||
|
normalized = row["normalized_attributes"] or {}
|
||||||
|
keys = set(raw) | set(normalized)
|
||||||
|
for key in keys:
|
||||||
|
bind.execute(
|
||||||
|
sa.text(
|
||||||
|
"""
|
||||||
|
INSERT INTO product_observation_attributes
|
||||||
|
(id, observation_id, attribute_key, raw_value, normalized_value, unit)
|
||||||
|
VALUES
|
||||||
|
(gen_random_uuid(), :observation_id, :attribute_key, :raw_value, :normalized_value, NULL)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"observation_id": row["id"],
|
||||||
|
"attribute_key": key,
|
||||||
|
"raw_value": raw.get(key),
|
||||||
|
"normalized_value": normalized.get(key),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.add_column("product_observations", sa.Column("raw_attributes", postgresql.JSONB, nullable=False, server_default="{}"))
|
||||||
|
op.add_column(
|
||||||
|
"product_observations",
|
||||||
|
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False, server_default="{}"),
|
||||||
|
)
|
||||||
|
op.add_column("buying_project_criteria", sa.Column("definition", postgresql.JSONB, nullable=False, server_default='{"items":[]}'))
|
||||||
|
op.drop_table("product_observation_attributes")
|
||||||
|
op.drop_table("project_criteria")
|
||||||
|
op.drop_column("source_categories", "canonical_category")
|
||||||
|
op.drop_column("criterion_definitions", "category")
|
||||||
261
backend/migrations/versions/0005_frontend_mvp_v03.py
Normal file
261
backend/migrations/versions/0005_frontend_mvp_v03.py
Normal file
|
|
@ -0,0 +1,261 @@
|
||||||
|
"""Frontend MVP v0.3 contracts: categories, optional templates, criterion UUIDs.
|
||||||
|
|
||||||
|
Revision ID: 0005_frontend_mvp_v03
|
||||||
|
Revises: 0004_dynamic_attributes
|
||||||
|
Create Date: 2026-10-04
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0005_frontend_mvp_v03"
|
||||||
|
down_revision: str | None = "0004_dynamic_attributes"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"product_categories",
|
||||||
|
sa.Column("id", sa.String(64), primary_key=True),
|
||||||
|
sa.Column("name", sa.String(256), nullable=False),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False, server_default="ACTIVE"),
|
||||||
|
)
|
||||||
|
op.execute("INSERT INTO product_categories (id, name, status) VALUES ('notebook', 'Notebooks', 'ACTIVE')")
|
||||||
|
|
||||||
|
op.add_column("source_categories", sa.Column("product_category_id", sa.String(64)))
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_source_categories_product_category",
|
||||||
|
"source_categories",
|
||||||
|
"product_categories",
|
||||||
|
["product_category_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE source_categories
|
||||||
|
SET product_category_id = canonical_category
|
||||||
|
WHERE canonical_category IS NOT NULL
|
||||||
|
AND EXISTS (SELECT 1 FROM product_categories WHERE id = source_categories.canonical_category)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column("criterion_definitions", sa.Column("id", postgresql.UUID(as_uuid=True)))
|
||||||
|
op.execute("UPDATE criterion_definitions SET id = gen_random_uuid() WHERE id IS NULL")
|
||||||
|
op.alter_column("criterion_definitions", "id", nullable=False)
|
||||||
|
op.add_column(
|
||||||
|
"criterion_definitions",
|
||||||
|
sa.Column("origin", sa.String(16), nullable=False, server_default="SOURCE"),
|
||||||
|
)
|
||||||
|
op.add_column("criterion_definitions", sa.Column("source", sa.String(64)))
|
||||||
|
op.add_column("criterion_definitions", sa.Column("product_category_id", sa.String(64)))
|
||||||
|
op.add_column("criterion_definitions", sa.Column("label", sa.String(256)))
|
||||||
|
op.add_column("criterion_definitions", sa.Column("scoring_attribute_key", sa.String(128)))
|
||||||
|
op.execute("UPDATE criterion_definitions SET label = key WHERE label IS NULL")
|
||||||
|
op.execute("UPDATE criterion_definitions SET scoring_attribute_key = key")
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE criterion_definitions
|
||||||
|
SET product_category_id = category
|
||||||
|
WHERE category IS NOT NULL
|
||||||
|
AND EXISTS (SELECT 1 FROM product_categories WHERE id = criterion_definitions.category)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.alter_column("criterion_definitions", "label", nullable=False)
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_criterion_definitions_product_category",
|
||||||
|
"criterion_definitions",
|
||||||
|
"product_categories",
|
||||||
|
["product_category_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column("project_criteria", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
|
||||||
|
op.add_column(
|
||||||
|
"project_criteria",
|
||||||
|
sa.Column("enabled_for_discovery", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||||
|
)
|
||||||
|
op.add_column("project_criteria", sa.Column("created_at", sa.DateTime(timezone=True)))
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE project_criteria
|
||||||
|
SET enabled_for_discovery = mandatory,
|
||||||
|
created_at = timezone('utc', now())
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE project_criteria AS pc
|
||||||
|
SET criterion_definition_id = cd.id
|
||||||
|
FROM criterion_definitions AS cd
|
||||||
|
WHERE pc.criterion_key = cd.key
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.alter_column("project_criteria", "criterion_definition_id", nullable=False)
|
||||||
|
op.alter_column("project_criteria", "created_at", nullable=False)
|
||||||
|
op.drop_constraint("project_criteria_criterion_key_fkey", "project_criteria", type_="foreignkey")
|
||||||
|
|
||||||
|
op.add_column("criterion_mappings", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE criterion_mappings AS cm
|
||||||
|
SET criterion_definition_id = cd.id
|
||||||
|
FROM criterion_definitions AS cd
|
||||||
|
WHERE cm.criterion_key = cd.key
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.alter_column("criterion_mappings", "criterion_definition_id", nullable=False)
|
||||||
|
op.drop_constraint("criterion_mappings_criterion_key_fkey", "criterion_mappings", type_="foreignkey")
|
||||||
|
|
||||||
|
op.drop_constraint("criterion_definitions_pkey", "criterion_definitions", type_="primary")
|
||||||
|
op.create_primary_key("criterion_definitions_pkey", "criterion_definitions", ["id"])
|
||||||
|
op.create_unique_constraint("uq_criterion_definition_key", "criterion_definitions", ["key"])
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_project_criteria_definition",
|
||||||
|
"project_criteria",
|
||||||
|
"criterion_definitions",
|
||||||
|
["criterion_definition_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_criterion_mappings_definition",
|
||||||
|
"criterion_mappings",
|
||||||
|
"criterion_definitions",
|
||||||
|
["criterion_definition_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"source_attribute_mappings",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||||
|
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id")),
|
||||||
|
sa.Column("raw_name", sa.String(256), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"criterion_definition_id",
|
||||||
|
postgresql.UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("criterion_definitions.id"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column("buying_projects", sa.Column("product_category_id", sa.String(64)))
|
||||||
|
op.add_column("buying_projects", sa.Column("market", sa.String(8)))
|
||||||
|
op.add_column("buying_projects", sa.Column("currency", sa.String(3)))
|
||||||
|
op.add_column("buying_projects", sa.Column("target_budget", sa.Numeric(12, 2)))
|
||||||
|
op.add_column("buying_projects", sa.Column("template_id", sa.String(128)))
|
||||||
|
op.add_column("buying_projects", sa.Column("template_version", sa.Integer()))
|
||||||
|
op.add_column("buying_projects", sa.Column("criteria_version", sa.Integer(), nullable=False, server_default="1"))
|
||||||
|
op.add_column("buying_projects", sa.Column("updated_at", sa.DateTime(timezone=True)))
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE buying_projects
|
||||||
|
SET template_id = profile_id,
|
||||||
|
template_version = profile_version,
|
||||||
|
product_category_id = 'notebook',
|
||||||
|
market = 'DE',
|
||||||
|
currency = 'EUR',
|
||||||
|
updated_at = created_at
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE buying_projects AS bp
|
||||||
|
SET criteria_version = COALESCE((
|
||||||
|
SELECT MAX(version) FROM buying_project_criteria WHERE project_id = bp.id
|
||||||
|
), 1)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.alter_column("buying_projects", "product_category_id", nullable=False)
|
||||||
|
op.alter_column("buying_projects", "market", nullable=False)
|
||||||
|
op.alter_column("buying_projects", "currency", nullable=False)
|
||||||
|
op.alter_column("buying_projects", "updated_at", nullable=False)
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_buying_projects_product_category",
|
||||||
|
"buying_projects",
|
||||||
|
"product_categories",
|
||||||
|
["product_category_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
op.drop_constraint("fk_buying_projects_profile", "buying_projects", type_="foreignkey")
|
||||||
|
op.drop_constraint("ck_buying_projects_interval", "buying_projects", type_="check")
|
||||||
|
op.drop_column("buying_projects", "profile_id")
|
||||||
|
op.drop_column("buying_projects", "profile_version")
|
||||||
|
op.drop_column("buying_projects", "schedule_interval_minutes")
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_buying_projects_template",
|
||||||
|
"buying_projects",
|
||||||
|
"product_profiles",
|
||||||
|
["template_id", "template_version"],
|
||||||
|
["id", "version"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"discovery_searches",
|
||||||
|
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
||||||
|
)
|
||||||
|
op.add_column("search_runs", sa.Column("new_count", sa.Integer(), nullable=False, server_default="0"))
|
||||||
|
op.add_column("search_runs", sa.Column("changed_count", sa.Integer(), nullable=False, server_default="0"))
|
||||||
|
op.add_column("search_runs", sa.Column("unchanged_count", sa.Integer(), nullable=False, server_default="0"))
|
||||||
|
op.add_column("search_runs", sa.Column("missing_count", sa.Integer(), nullable=False, server_default="0"))
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"product_observation_attributes",
|
||||||
|
sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)),
|
||||||
|
)
|
||||||
|
op.add_column("product_observation_attributes", sa.Column("raw_name", sa.String(256)))
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_observation_attributes_definition",
|
||||||
|
"product_observation_attributes",
|
||||||
|
"criterion_definitions",
|
||||||
|
["criterion_definition_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE product_observation_attributes AS poa
|
||||||
|
SET criterion_definition_id = cd.id,
|
||||||
|
raw_name = poa.attribute_key
|
||||||
|
FROM criterion_definitions AS cd
|
||||||
|
WHERE poa.attribute_key = cd.scoring_attribute_key
|
||||||
|
OR poa.attribute_key = cd.key
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint("fk_observation_attributes_definition", "product_observation_attributes", type_="foreignkey")
|
||||||
|
op.drop_column("product_observation_attributes", "raw_name")
|
||||||
|
op.drop_column("product_observation_attributes", "criterion_definition_id")
|
||||||
|
op.drop_column("search_runs", "missing_count")
|
||||||
|
op.drop_column("search_runs", "unchanged_count")
|
||||||
|
op.drop_column("search_runs", "changed_count")
|
||||||
|
op.drop_column("search_runs", "new_count")
|
||||||
|
op.drop_column("discovery_searches", "schedule_interval_minutes")
|
||||||
|
op.drop_constraint("fk_buying_projects_template", "buying_projects", type_="foreignkey")
|
||||||
|
op.add_column("buying_projects", sa.Column("profile_id", sa.String(128)))
|
||||||
|
op.add_column("buying_projects", sa.Column("profile_version", sa.Integer()))
|
||||||
|
op.add_column(
|
||||||
|
"buying_projects",
|
||||||
|
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
||||||
|
)
|
||||||
|
op.execute("UPDATE buying_projects SET profile_id = template_id, profile_version = template_version")
|
||||||
|
op.alter_column("buying_projects", "profile_id", nullable=False)
|
||||||
|
op.alter_column("buying_projects", "profile_version", nullable=False)
|
||||||
|
op.drop_constraint("fk_buying_projects_product_category", "buying_projects", type_="foreignkey")
|
||||||
|
op.drop_column("buying_projects", "updated_at")
|
||||||
|
op.drop_column("buying_projects", "criteria_version")
|
||||||
|
op.drop_column("buying_projects", "template_version")
|
||||||
|
op.drop_column("buying_projects", "template_id")
|
||||||
|
op.drop_column("buying_projects", "target_budget")
|
||||||
|
op.drop_column("buying_projects", "currency")
|
||||||
|
op.drop_column("buying_projects", "market")
|
||||||
|
op.drop_column("buying_projects", "product_category_id")
|
||||||
|
op.drop_table("source_attribute_mappings")
|
||||||
|
raise NotImplementedError("full downgrade of criterion UUID primary key is not supported")
|
||||||
23
backend/profiles/mobile-development-notebook.json
Normal file
23
backend/profiles/mobile-development-notebook.json
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
{
|
||||||
|
"id": "mobile-development-notebook",
|
||||||
|
"version": 1,
|
||||||
|
"name": "Mobiler Entwicklungsrechner",
|
||||||
|
"category": "notebook",
|
||||||
|
"currency": "EUR",
|
||||||
|
"market": "DE",
|
||||||
|
"allowed_conditions": ["new"],
|
||||||
|
"budget": {"target": "1500.00", "maximum": "2000.00"},
|
||||||
|
"scoring_version": "generic-v1",
|
||||||
|
"constraints": [
|
||||||
|
{"id": "min_ram", "attribute": "ram_gb", "operator": ">=", "value": 16, "mandatory": true},
|
||||||
|
{"id": "min_storage", "attribute": "storage_gb", "operator": ">=", "value": 512, "mandatory": true},
|
||||||
|
{"id": "max_weight", "attribute": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": true}
|
||||||
|
],
|
||||||
|
"preferences": [
|
||||||
|
{"id": "ram", "attribute": "ram_gb", "strategy": "numeric_linear", "direction": "higher", "acceptable": 16, "ideal": 32, "weight": 0.30},
|
||||||
|
{"id": "weight", "attribute": "weight_kg", "strategy": "numeric_linear", "direction": "lower", "acceptable": 1.6, "ideal": 1.0, "weight": 0.25},
|
||||||
|
{"id": "battery", "attribute": "battery_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.20},
|
||||||
|
{"id": "display", "attribute": "display_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.15},
|
||||||
|
{"id": "cpu", "attribute": "cpu_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.10}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -2,7 +2,6 @@
|
||||||
name = "buying-agent"
|
name = "buying-agent"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "Generic buying and deal evaluation agent"
|
description = "Generic buying and deal evaluation agent"
|
||||||
readme = "README.md"
|
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"alembic>=1.16",
|
"alembic>=1.16",
|
||||||
|
|
@ -25,7 +24,7 @@ requires = ["setuptools>=75"]
|
||||||
build-backend = "setuptools.build_meta"
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
include = ["app*"]
|
include = ["app*", "scripts*"]
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
testpaths = ["tests"]
|
testpaths = ["tests"]
|
||||||
|
|
@ -33,4 +32,5 @@ addopts = "-ra"
|
||||||
markers = [
|
markers = [
|
||||||
"slow: long-running tests skipped in the default CI suite",
|
"slow: long-running tests skipped in the default CI suite",
|
||||||
"integration: needs PostgreSQL 17",
|
"integration: needs PostgreSQL 17",
|
||||||
|
"e2e: HTTP end-to-end against PostgreSQL 17",
|
||||||
]
|
]
|
||||||
23
backend/rules/generic-v1.json
Normal file
23
backend/rules/generic-v1.json
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
{
|
||||||
|
"version": "generic-v1",
|
||||||
|
"algorithm_id": "linear-budget-history-v1",
|
||||||
|
"supported_preference_strategies": ["numeric_linear"],
|
||||||
|
"minimum_confidence": 0.8,
|
||||||
|
"maximum_price_age_hours": 48,
|
||||||
|
"optional_constraint_penalty_points": 5,
|
||||||
|
"missing_preference_score": 0,
|
||||||
|
"product_weight": 0.6,
|
||||||
|
"price_weight": 0.4,
|
||||||
|
"budget_weight": 0.4,
|
||||||
|
"history_weight": 0.6,
|
||||||
|
"history_window_days": 90,
|
||||||
|
"minimum_history_days": 3,
|
||||||
|
"history_discount_for_max_score": 0.2,
|
||||||
|
"classification_upper_bounds": [
|
||||||
|
{"upper": 60, "inclusive": false, "label": "IGNORE"},
|
||||||
|
{"upper": 75, "inclusive": false, "label": "WATCH"},
|
||||||
|
{"upper": 85, "inclusive": false, "label": "GOOD"},
|
||||||
|
{"upper": 92, "inclusive": true, "label": "VERY_GOOD"},
|
||||||
|
{"upper": 100, "inclusive": true, "label": "EXCEPTIONAL"}
|
||||||
|
]
|
||||||
|
}
|
||||||
0
backend/scripts/__init__.py
Normal file
0
backend/scripts/__init__.py
Normal file
71
backend/scripts/bootstrap.py
Normal file
71
backend/scripts/bootstrap.py
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.domain.validation import parse_scoring_model
|
||||||
|
from app.persistence.db import SessionLocal
|
||||||
|
from app.persistence.models import ScoringModelRecord, SourceRecord
|
||||||
|
from app.services.catalog_service import ensure_product_category
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
RULES = ROOT / "rules" / "generic-v1.json"
|
||||||
|
|
||||||
|
|
||||||
|
def run_bootstrap(session: Session) -> None:
|
||||||
|
_seed_source(session)
|
||||||
|
_seed_scoring_model(session)
|
||||||
|
ensure_product_category(session, "notebook", "Notebooks")
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_source(session: Session) -> None:
|
||||||
|
_upsert_source(session, "manual", "Manual entry", "manual")
|
||||||
|
_upsert_source(session, "geizhals", "Geizhals", "geizhals")
|
||||||
|
|
||||||
|
|
||||||
|
def _upsert_source(session: Session, source_id: str, name: str, adapter_type: str) -> None:
|
||||||
|
existing = session.get(SourceRecord, source_id)
|
||||||
|
if existing is None:
|
||||||
|
session.add(SourceRecord(id=source_id, name=name, adapter_type=adapter_type))
|
||||||
|
return
|
||||||
|
if existing.adapter_type != adapter_type:
|
||||||
|
raise SystemExit(f"bootstrap conflict: source '{source_id}' has a different adapter_type")
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_scoring_model(session: Session) -> None:
|
||||||
|
definition = json.loads(RULES.read_text(encoding="utf-8"))
|
||||||
|
model = parse_scoring_model(definition)
|
||||||
|
existing = session.get(ScoringModelRecord, model.version)
|
||||||
|
if existing is None:
|
||||||
|
session.add(
|
||||||
|
ScoringModelRecord(
|
||||||
|
version=model.version,
|
||||||
|
algorithm_id=model.algorithm_id,
|
||||||
|
definition=model.definition,
|
||||||
|
definition_hash=model.definition_hash,
|
||||||
|
created_at=datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if existing.definition_hash != model.definition_hash:
|
||||||
|
raise SystemExit(
|
||||||
|
f"bootstrap conflict: scoring model {model.version} hash "
|
||||||
|
f"{existing.definition_hash} != {model.definition_hash}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
session = SessionLocal()
|
||||||
|
try:
|
||||||
|
run_bootstrap(session)
|
||||||
|
print("bootstrap ok")
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
138
backend/scripts/demo.py
Normal file
138
backend/scripts/demo.py
Normal file
|
|
@ -0,0 +1,138 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
PROFILE = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
|
||||||
|
FIXTURE = json.loads((ROOT / "fixtures" / "manual-demo.json").read_text(encoding="utf-8"))
|
||||||
|
BASE = "http://127.0.0.1:8000"
|
||||||
|
AS_OF = "2026-10-03T12:00:00Z"
|
||||||
|
|
||||||
|
DEFINITIONS = [
|
||||||
|
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0, "description": "RAM in GB"},
|
||||||
|
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0, "description": "Storage in GB"},
|
||||||
|
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0, "description": "Weight in kg"},
|
||||||
|
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual battery score"},
|
||||||
|
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual display score"},
|
||||||
|
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual CPU score"},
|
||||||
|
{"key": "ports", "value_type": "string_list", "description": "Ports"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def request(method: str, path: str, body=None, expected=(200, 201, 409)):
|
||||||
|
data = None if body is None else json.dumps(body, default=str).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{BASE}{path}",
|
||||||
|
data=data,
|
||||||
|
method=method,
|
||||||
|
headers={"Content-Type": "application/json"} if body is not None else {},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
payload = json.loads(response.read().decode())
|
||||||
|
if response.status not in expected:
|
||||||
|
raise SystemExit(f"{method} {path} -> {response.status}: {payload}")
|
||||||
|
return response.status, payload
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
payload = json.loads(exc.read().decode())
|
||||||
|
if exc.code not in expected:
|
||||||
|
raise SystemExit(f"{method} {path} -> {exc.code}: {payload}") from exc
|
||||||
|
return exc.code, payload
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
for definition in DEFINITIONS:
|
||||||
|
request("POST", "/api/v1/attribute-definitions", definition)
|
||||||
|
request("POST", "/api/v1/profiles", PROFILE)
|
||||||
|
|
||||||
|
product = FIXTURE["products"][0]
|
||||||
|
request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/products",
|
||||||
|
{
|
||||||
|
"id": product["id"],
|
||||||
|
"category": product["category"],
|
||||||
|
"brand": product["brand"],
|
||||||
|
"name": product["name"],
|
||||||
|
"model": product.get("model"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for variant in product["variants"]:
|
||||||
|
request(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/products/{product['id']}/variants",
|
||||||
|
{
|
||||||
|
"id": variant["id"],
|
||||||
|
"name": variant["name"],
|
||||||
|
"manufacturer_sku": variant.get("manufacturer_sku"),
|
||||||
|
"identifiers": variant.get("identifiers") or {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for attribute in variant["attributes"]:
|
||||||
|
request("POST", f"/api/v1/variants/{variant['id']}/attributes", {**attribute, "source_id": "manual"})
|
||||||
|
|
||||||
|
for offer in FIXTURE["offers"]:
|
||||||
|
request(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/variants/{offer['variant_id']}/offers",
|
||||||
|
{
|
||||||
|
"id": offer["id"],
|
||||||
|
"source_id": "manual",
|
||||||
|
"external_id": offer["external_id"],
|
||||||
|
"seller": offer["seller"],
|
||||||
|
"url": offer.get("url"),
|
||||||
|
"market": offer["market"],
|
||||||
|
"currency": offer["currency"],
|
||||||
|
"condition": offer["condition"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for price in FIXTURE["prices"]:
|
||||||
|
offer = next(item for item in FIXTURE["offers"] if item["external_id"] == price["external_offer_id"])
|
||||||
|
request(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/offers/{offer['id']}/prices",
|
||||||
|
{
|
||||||
|
"id": price["id"],
|
||||||
|
"source_id": "manual",
|
||||||
|
"external_observation_id": price["external_observation_id"],
|
||||||
|
"observed_at": price["observed_at"],
|
||||||
|
"item_price": price["item_price"],
|
||||||
|
"shipping_price": price["shipping_price"],
|
||||||
|
"currency": price["currency"],
|
||||||
|
"availability": price["availability"],
|
||||||
|
"raw_payload": price["raw_payload"],
|
||||||
|
"confidence": price["confidence"],
|
||||||
|
},
|
||||||
|
expected=(200, 201),
|
||||||
|
)
|
||||||
|
|
||||||
|
cases = {
|
||||||
|
"A": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-333333333333"),
|
||||||
|
"B": ("22222222-2222-4222-8222-22222222222b", "33333333-3333-4333-8333-33333333333b"),
|
||||||
|
"C": ("22222222-2222-4222-8222-22222222222c", "33333333-3333-4333-8333-33333333333c"),
|
||||||
|
"D": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-33333333333d"),
|
||||||
|
}
|
||||||
|
for label, (variant_id, offer_id) in cases.items():
|
||||||
|
status, result = request(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/evaluate/{variant_id}",
|
||||||
|
{
|
||||||
|
"profile_id": "mobile-development-notebook",
|
||||||
|
"profile_version": 1,
|
||||||
|
"offer_id": offer_id,
|
||||||
|
"as_of": AS_OF,
|
||||||
|
},
|
||||||
|
expected=(201,),
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"case {label}: {status} overall={result['overall_eligibility']} "
|
||||||
|
f"product={result['product_score']} price={result['price_score']} "
|
||||||
|
f"deal={result['deal_score']} class={result['classification']}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
81
backend/scripts/seed_notebook_pilot.py
Normal file
81
backend/scripts/seed_notebook_pilot.py
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
"""Notebook-pilot metadata only. Not part of the core schema or bootstrap."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.discovery.definitions import get_definition_by_key, upsert_definition
|
||||||
|
from app.persistence.models import AttributeDefinitionRecord
|
||||||
|
from app.services.catalog_service import ensure_product_category
|
||||||
|
|
||||||
|
|
||||||
|
NOTEBOOK_ATTRIBUTE_SEEDS = (
|
||||||
|
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
||||||
|
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
||||||
|
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
|
||||||
|
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
)
|
||||||
|
|
||||||
|
NOTEBOOK_CRITERION_SEEDS = (
|
||||||
|
{
|
||||||
|
"key": "ram_gb",
|
||||||
|
"origin": "SOURCE",
|
||||||
|
"source": "geizhals",
|
||||||
|
"product_category_id": "notebook",
|
||||||
|
"label": "Arbeitsspeicher",
|
||||||
|
"value_type": "number",
|
||||||
|
"unit": "GB",
|
||||||
|
"semantic": "working_memory",
|
||||||
|
"category": "notebook",
|
||||||
|
"scoring_attribute_key": "ram_gb",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "storage_gb",
|
||||||
|
"origin": "SOURCE",
|
||||||
|
"source": "geizhals",
|
||||||
|
"product_category_id": "notebook",
|
||||||
|
"label": "SSD",
|
||||||
|
"value_type": "number",
|
||||||
|
"unit": "GB",
|
||||||
|
"semantic": "storage",
|
||||||
|
"category": "notebook",
|
||||||
|
"scoring_attribute_key": "storage_gb",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "weight_kg",
|
||||||
|
"origin": "SOURCE",
|
||||||
|
"source": "geizhals",
|
||||||
|
"product_category_id": "notebook",
|
||||||
|
"label": "Gewicht",
|
||||||
|
"value_type": "number",
|
||||||
|
"unit": "kg",
|
||||||
|
"semantic": "weight",
|
||||||
|
"category": "notebook",
|
||||||
|
"scoring_attribute_key": "weight_kg",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "price_eur",
|
||||||
|
"origin": "SOURCE",
|
||||||
|
"source": "geizhals",
|
||||||
|
"product_category_id": "notebook",
|
||||||
|
"label": "Preis",
|
||||||
|
"value_type": "number",
|
||||||
|
"unit": "EUR",
|
||||||
|
"semantic": "price",
|
||||||
|
"category": "notebook",
|
||||||
|
"scoring_attribute_key": "price_eur",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def seed_notebook_pilot(session: Session) -> None:
|
||||||
|
ensure_product_category(session, "notebook", "Notebooks")
|
||||||
|
for item in NOTEBOOK_ATTRIBUTE_SEEDS:
|
||||||
|
if session.get(AttributeDefinitionRecord, item["key"]) is None:
|
||||||
|
session.add(AttributeDefinitionRecord(**item, description=None))
|
||||||
|
for item in NOTEBOOK_CRITERION_SEEDS:
|
||||||
|
if get_definition_by_key(session, item["key"]) is None:
|
||||||
|
upsert_definition(session, item)
|
||||||
|
session.flush()
|
||||||
|
|
@ -3,5 +3,6 @@ set -e
|
||||||
|
|
||||||
echo "=== BuyingAgent API startup ==="
|
echo "=== BuyingAgent API startup ==="
|
||||||
alembic upgrade head
|
alembic upgrade head
|
||||||
|
python -m scripts.bootstrap
|
||||||
echo "=== starting uvicorn ==="
|
echo "=== starting uvicorn ==="
|
||||||
exec uvicorn app.main:app --host 0.0.0.0 --port 8000
|
exec uvicorn app.main:app --host 0.0.0.0 --port 8000
|
||||||
0
backend/tests/__init__.py
Normal file
0
backend/tests/__init__.py
Normal file
150
backend/tests/conftest.py
Normal file
150
backend/tests/conftest.py
Normal file
|
|
@ -0,0 +1,150 @@
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from decimal import Decimal
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
|
||||||
|
from app.domain.models import (
|
||||||
|
AttributeDefinition,
|
||||||
|
AttributeStatement,
|
||||||
|
Offer,
|
||||||
|
PriceObservation,
|
||||||
|
Product,
|
||||||
|
ProductVariant,
|
||||||
|
)
|
||||||
|
from app.domain.validation import parse_profile, parse_scoring_model
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
AS_OF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)
|
||||||
|
CUTOFF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def scoring_model():
|
||||||
|
definition = json.loads((ROOT / "rules" / "generic-v1.json").read_text(encoding="utf-8"))
|
||||||
|
return parse_scoring_model(definition)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def definitions() -> dict[str, AttributeDefinition]:
|
||||||
|
return {
|
||||||
|
"ram_gb": AttributeDefinition("ram_gb", ValueType.NUMBER, "GB", Decimal("0"), None, None),
|
||||||
|
"storage_gb": AttributeDefinition("storage_gb", ValueType.NUMBER, "GB", Decimal("0"), None, None),
|
||||||
|
"weight_kg": AttributeDefinition("weight_kg", ValueType.NUMBER, "kg", Decimal("0"), None, None),
|
||||||
|
"battery_score": AttributeDefinition("battery_score", ValueType.NUMBER, None, Decimal("0"), Decimal("100"), None),
|
||||||
|
"display_score": AttributeDefinition("display_score", ValueType.NUMBER, None, Decimal("0"), Decimal("100"), None),
|
||||||
|
"cpu_score": AttributeDefinition("cpu_score", ValueType.NUMBER, None, Decimal("0"), Decimal("100"), None),
|
||||||
|
"ports": AttributeDefinition("ports", ValueType.STRING_LIST, None, None, None, None),
|
||||||
|
"color": AttributeDefinition("color", ValueType.STRING, None, None, None, None),
|
||||||
|
"has_lte": AttributeDefinition("has_lte", ValueType.BOOLEAN, None, None, None, None),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def profile(scoring_model, definitions):
|
||||||
|
raw = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
|
||||||
|
return parse_profile(
|
||||||
|
raw,
|
||||||
|
attribute_definitions=definitions,
|
||||||
|
scoring_versions={scoring_model.version},
|
||||||
|
supported_strategies=set(scoring_model.supported_preference_strategies),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def utc(year, month, day, hour=10, minute=0) -> datetime:
|
||||||
|
return datetime(year, month, day, hour, minute, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def statement(attribute: str, value, *, confidence="1.0", status="normalized", observed=None, ingested=None, row_id=None):
|
||||||
|
return AttributeStatement(
|
||||||
|
id=row_id or uuid4(),
|
||||||
|
variant_id=UUID("22222222-2222-4222-8222-222222222222"),
|
||||||
|
attribute=attribute,
|
||||||
|
raw_value=value,
|
||||||
|
normalized_value=None if status != "normalized" else value,
|
||||||
|
unit="GB" if attribute.endswith("_gb") else ("kg" if attribute.endswith("_kg") else None),
|
||||||
|
normalization_status=NormalizationStatus(status),
|
||||||
|
normalizer_version="manual-v1",
|
||||||
|
confidence=Decimal(str(confidence)),
|
||||||
|
source_id="manual",
|
||||||
|
source_reference=f"test/{attribute}",
|
||||||
|
observed_at=observed or utc(2026, 10, 3, 9),
|
||||||
|
ingested_at=ingested or utc(2026, 10, 3, 9, 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def product_a() -> Product:
|
||||||
|
return Product(
|
||||||
|
id=UUID("11111111-1111-4111-8111-111111111111"),
|
||||||
|
category="notebook",
|
||||||
|
brand="Example",
|
||||||
|
name="Example Mobile 13",
|
||||||
|
model="EM13",
|
||||||
|
created_at=utc(2026, 10, 1, 8),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def variant_a() -> ProductVariant:
|
||||||
|
return ProductVariant(
|
||||||
|
id=UUID("22222222-2222-4222-8222-222222222222"),
|
||||||
|
product_id=UUID("11111111-1111-4111-8111-111111111111"),
|
||||||
|
name="32/1024",
|
||||||
|
manufacturer_sku="EM13-32-1T",
|
||||||
|
identifiers={},
|
||||||
|
created_at=utc(2026, 10, 1, 8),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def offer_a() -> Offer:
|
||||||
|
return Offer(
|
||||||
|
id=UUID("33333333-3333-4333-8333-333333333333"),
|
||||||
|
variant_id=UUID("22222222-2222-4222-8222-222222222222"),
|
||||||
|
source_id="manual",
|
||||||
|
external_id="demo-offer-a",
|
||||||
|
seller="DemoShop",
|
||||||
|
url=None,
|
||||||
|
market="DE",
|
||||||
|
currency="EUR",
|
||||||
|
condition=OfferCondition.NEW,
|
||||||
|
created_at=utc(2026, 10, 1, 8),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def observation(day, total, *, offer_id=None, obs_id=None, availability="in_stock", confidence="1.0", hour=10):
|
||||||
|
return PriceObservation(
|
||||||
|
id=obs_id or uuid4(),
|
||||||
|
offer_id=offer_id or UUID("33333333-3333-4333-8333-333333333333"),
|
||||||
|
source_id="manual",
|
||||||
|
external_observation_id=str(obs_id or uuid4()),
|
||||||
|
observed_at=utc(2026, day[0], day[1], hour) if isinstance(day, tuple) else day,
|
||||||
|
ingested_at=utc(2026, 10, 3, 11),
|
||||||
|
item_price=Decimal(str(total)),
|
||||||
|
shipping_price=Decimal("0.00"),
|
||||||
|
currency="EUR",
|
||||||
|
availability=Availability(availability),
|
||||||
|
raw_payload={"total": str(total)},
|
||||||
|
confidence=Decimal(str(confidence)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def case_a_attributes():
|
||||||
|
return (
|
||||||
|
statement("ram_gb", 32),
|
||||||
|
statement("storage_gb", 1024),
|
||||||
|
statement("weight_kg", Decimal("1.2")),
|
||||||
|
statement("battery_score", 90),
|
||||||
|
statement("display_score", 90),
|
||||||
|
statement("cpu_score", 90),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def case_a_prices():
|
||||||
|
return (
|
||||||
|
observation((9, 1), "1599.00"),
|
||||||
|
observation((9, 15), "1599.00"),
|
||||||
|
observation((10, 1), "1599.00"),
|
||||||
|
observation((10, 3), "1298.00"),
|
||||||
|
)
|
||||||
0
backend/tests/e2e/__init__.py
Normal file
0
backend/tests/e2e/__init__.py
Normal file
199
backend/tests/e2e/test_discovery_walkthrough.py
Normal file
199
backend/tests/e2e/test_discovery_walkthrough.py
Normal file
|
|
@ -0,0 +1,199 @@
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.main import app
|
||||||
|
from app.persistence.db import SessionLocal, engine
|
||||||
|
from scripts.bootstrap import run_bootstrap
|
||||||
|
from scripts.seed_notebook_pilot import seed_notebook_pilot
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.e2e, pytest.mark.integration]
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
PROFILE = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
DEFINITIONS = [
|
||||||
|
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
||||||
|
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
||||||
|
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
|
||||||
|
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
]
|
||||||
|
|
||||||
|
CRITERIA = {
|
||||||
|
"items": [
|
||||||
|
{"id": "ram", "criterion": "ram_gb", "operator": ">=", "value": 24, "mandatory": True},
|
||||||
|
{"id": "price", "criterion": "price_eur", "operator": "<=", "value": 2000, "mandatory": True},
|
||||||
|
{"id": "weight", "criterion": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": True},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def require_postgres() -> None:
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
connection.execute(text("SELECT 1"))
|
||||||
|
except Exception as exc:
|
||||||
|
pytest.skip(f"PostgreSQL not available: {exc}")
|
||||||
|
session = SessionLocal()
|
||||||
|
try:
|
||||||
|
run_bootstrap(session)
|
||||||
|
seed_notebook_pilot(session)
|
||||||
|
session.commit()
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client() -> TestClient:
|
||||||
|
return TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def _ok(response, allowed=(200, 201, 409)) -> None:
|
||||||
|
assert response.status_code in allowed, response.text
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_profile(client: TestClient) -> None:
|
||||||
|
for definition in DEFINITIONS:
|
||||||
|
_ok(client.post("/api/v1/attribute-definitions", json=definition))
|
||||||
|
_ok(client.post("/api/v1/profiles", json=PROFILE))
|
||||||
|
|
||||||
|
|
||||||
|
def test_notebook_discovery_pilot(client: TestClient) -> None:
|
||||||
|
_seed_profile(client)
|
||||||
|
taxonomy = client.post("/api/v1/sources/geizhals/categories/nb/discover-taxonomy")
|
||||||
|
assert taxonomy.status_code == 201, taxonomy.text
|
||||||
|
category_id = taxonomy.json()["id"]
|
||||||
|
filters = client.get("/api/v1/sources/geizhals/categories/nb/filters")
|
||||||
|
assert filters.status_code == 200
|
||||||
|
assert {item["external_filter_key"] for item in filters.json()["filters"]} >= {"17023", "price"}
|
||||||
|
|
||||||
|
ram = client.post(
|
||||||
|
"/api/v1/criterion-mappings",
|
||||||
|
json={
|
||||||
|
"criterion_key": "ram_gb",
|
||||||
|
"source_id": "geizhals",
|
||||||
|
"source_category_id": category_id,
|
||||||
|
"external_filter_key": "17023",
|
||||||
|
"source_value": "ab 24GB",
|
||||||
|
"encoding": {"url_param": "xf"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert ram.status_code == 201, ram.text
|
||||||
|
assert ram.json()["status"] == "VERIFIED"
|
||||||
|
|
||||||
|
price = client.post(
|
||||||
|
"/api/v1/criterion-mappings",
|
||||||
|
json={
|
||||||
|
"criterion_key": "price_eur",
|
||||||
|
"source_id": "geizhals",
|
||||||
|
"source_category_id": category_id,
|
||||||
|
"external_filter_key": "price",
|
||||||
|
"source_value": None,
|
||||||
|
"encoding": {"url_param": "maxp"},
|
||||||
|
"probe_value": 2000,
|
||||||
|
"operator": "<=",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert price.status_code == 201, price.text
|
||||||
|
assert price.json()["status"] == "VERIFIED"
|
||||||
|
|
||||||
|
project = client.post(
|
||||||
|
"/api/v1/buying-projects",
|
||||||
|
json={
|
||||||
|
"name": "Notebook-Pilot",
|
||||||
|
"profile_id": "mobile-development-notebook",
|
||||||
|
"profile_version": 1,
|
||||||
|
"shortlist_size": 5,
|
||||||
|
"schedule_interval_minutes": 180,
|
||||||
|
"criteria": CRITERIA,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert project.status_code == 201, project.text
|
||||||
|
project_id = project.json()["id"]
|
||||||
|
assert project.json()["workspace_id"] is None
|
||||||
|
assert project.json()["criteria_version"] == 1
|
||||||
|
available = client.get(f"/api/v1/buying-projects/{project_id}/available-criteria")
|
||||||
|
assert available.status_code == 200, available.text
|
||||||
|
assert available.json()["product_category_id"] == "notebook"
|
||||||
|
scoring_keys = {item.get("scoring_attribute_key") for item in available.json()["criteria"]}
|
||||||
|
assert {"ram_gb", "price_eur"} <= scoring_keys
|
||||||
|
ram = next(item for item in available.json()["criteria"] if item.get("scoring_attribute_key") == "ram_gb")
|
||||||
|
assert ram["mapping_status"] == "VERIFIED"
|
||||||
|
|
||||||
|
search = client.post(
|
||||||
|
"/api/v1/discovery-searches",
|
||||||
|
json={
|
||||||
|
"buying_project_id": project_id,
|
||||||
|
"source_id": "geizhals",
|
||||||
|
"source_category_id": category_id,
|
||||||
|
"broad_result_threshold": 300,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert search.status_code == 201, search.text
|
||||||
|
search_id = search.json()["id"]
|
||||||
|
assert "xf=17023_ab+24GB" in search.json()["generated_url"]
|
||||||
|
assert "maxp=2000" in search.json()["generated_url"]
|
||||||
|
assert search.json()["schedule_interval_minutes"] == 180
|
||||||
|
|
||||||
|
due = client.get("/api/v1/discovery-searches?due=true")
|
||||||
|
assert due.status_code == 200
|
||||||
|
assert any(item["id"] == search_id for item in due.json()["items"])
|
||||||
|
|
||||||
|
run = client.post(f"/api/v1/discovery-searches/{search_id}/run")
|
||||||
|
assert run.status_code == 201, run.text
|
||||||
|
assert run.json()["status"] == "SUCCESS"
|
||||||
|
assert run.json()["page_count"] == 2
|
||||||
|
assert run.json()["result_count"] == 3
|
||||||
|
assert {item["diff_status"] for item in run.json()["items"]} == {"NEW"}
|
||||||
|
|
||||||
|
shortlist = client.get(f"/api/v1/buying-projects/{project_id}/shortlist")
|
||||||
|
assert shortlist.status_code == 200
|
||||||
|
assert len(shortlist.json()["items"]) == 1
|
||||||
|
assert shortlist.json()["items"][0]["product_eligibility"] == "ELIGIBLE"
|
||||||
|
assert shortlist.json()["items"][0]["status"] == "SHORTLISTED"
|
||||||
|
|
||||||
|
candidates = client.get(f"/api/v1/buying-projects/{project_id}/candidates")
|
||||||
|
assert candidates.status_code == 200
|
||||||
|
eligibilities = {item["product_eligibility"] for item in candidates.json()["items"]}
|
||||||
|
assert eligibilities == {"ELIGIBLE", "INELIGIBLE", "UNKNOWN"}
|
||||||
|
assert all(item.get("deal_score") is None for item in candidates.json()["items"])
|
||||||
|
|
||||||
|
updated = client.post(
|
||||||
|
f"/api/v1/buying-projects/{project_id}/criteria",
|
||||||
|
json={"items": CRITERIA["items"][:2]},
|
||||||
|
)
|
||||||
|
assert updated.status_code == 201
|
||||||
|
assert updated.json()["criteria_version"] == 2
|
||||||
|
stale = client.get(f"/api/v1/discovery-searches/{search_id}")
|
||||||
|
assert stale.json()["status"] == "OUT_OF_SYNC"
|
||||||
|
blocked_run = client.post(f"/api/v1/discovery-searches/{search_id}/run")
|
||||||
|
assert blocked_run.status_code == 409
|
||||||
|
refreshed = client.post(f"/api/v1/discovery-searches/{search_id}/generate-query")
|
||||||
|
assert refreshed.status_code == 200
|
||||||
|
assert refreshed.json()["status"] == "ACTIVE"
|
||||||
|
assert "weight_kg" not in refreshed.json().get("internal_only", [])
|
||||||
|
assert "weight_kg" not in refreshed.json()["generated_url"]
|
||||||
|
|
||||||
|
client.patch(f"/api/v1/discovery-searches/{search_id}", json={"status": "DISABLED"})
|
||||||
|
broad = client.post(
|
||||||
|
"/api/v1/discovery-searches",
|
||||||
|
json={
|
||||||
|
"buying_project_id": project_id,
|
||||||
|
"source_id": "geizhals",
|
||||||
|
"source_category_id": category_id,
|
||||||
|
"broad_result_threshold": 2,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert broad.status_code == 201, broad.text
|
||||||
|
broad_run = client.post(f"/api/v1/discovery-searches/{broad.json()['id']}/run")
|
||||||
|
assert broad_run.status_code == 201, broad_run.text
|
||||||
|
assert broad_run.json()["status"] == "TOO_BROAD"
|
||||||
|
assert broad_run.json()["result_count"] == 3
|
||||||
|
stored = client.get(f"/api/v1/discovery-searches/{broad.json()['id']}")
|
||||||
|
assert stored.json()["status"] == "TOO_BROAD"
|
||||||
204
backend/tests/e2e/test_http_walkthrough.py
Normal file
204
backend/tests/e2e/test_http_walkthrough.py
Normal file
|
|
@ -0,0 +1,204 @@
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.main import app
|
||||||
|
from app.persistence.db import SessionLocal, engine
|
||||||
|
from scripts.bootstrap import run_bootstrap
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.e2e, pytest.mark.integration]
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
PROFILE = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
|
||||||
|
FIXTURE = json.loads((ROOT / "fixtures" / "manual-demo.json").read_text(encoding="utf-8"))
|
||||||
|
AS_OF = "2026-10-03T12:00:00Z"
|
||||||
|
|
||||||
|
DEFINITIONS = [
|
||||||
|
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
||||||
|
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
||||||
|
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
|
||||||
|
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def require_postgres() -> None:
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
connection.execute(text("SELECT 1"))
|
||||||
|
except Exception as exc:
|
||||||
|
pytest.skip(f"PostgreSQL not available: {exc}")
|
||||||
|
session = SessionLocal()
|
||||||
|
try:
|
||||||
|
run_bootstrap(session)
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client() -> TestClient:
|
||||||
|
return TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def _ok(response, allowed=(200, 201, 409)) -> None:
|
||||||
|
assert response.status_code in allowed, response.text
|
||||||
|
|
||||||
|
|
||||||
|
def _seed(client: TestClient) -> None:
|
||||||
|
for definition in DEFINITIONS:
|
||||||
|
_ok(client.post("/api/v1/attribute-definitions", json=definition))
|
||||||
|
_ok(client.post("/api/v1/profiles", json=PROFILE))
|
||||||
|
product = FIXTURE["products"][0]
|
||||||
|
_ok(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/products",
|
||||||
|
json={
|
||||||
|
"id": product["id"],
|
||||||
|
"category": product["category"],
|
||||||
|
"brand": product["brand"],
|
||||||
|
"name": product["name"],
|
||||||
|
"model": product.get("model"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for variant in product["variants"]:
|
||||||
|
_ok(
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/products/{product['id']}/variants",
|
||||||
|
json={
|
||||||
|
"id": variant["id"],
|
||||||
|
"name": variant["name"],
|
||||||
|
"manufacturer_sku": variant.get("manufacturer_sku"),
|
||||||
|
"identifiers": variant.get("identifiers") or {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for attribute in variant["attributes"]:
|
||||||
|
_ok(client.post(f"/api/v1/variants/{variant['id']}/attributes", json={**attribute, "source_id": "manual"}))
|
||||||
|
for offer in FIXTURE["offers"]:
|
||||||
|
_ok(
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/variants/{offer['variant_id']}/offers",
|
||||||
|
json={
|
||||||
|
"id": offer["id"],
|
||||||
|
"source_id": "manual",
|
||||||
|
"external_id": offer["external_id"],
|
||||||
|
"seller": offer["seller"],
|
||||||
|
"url": offer.get("url"),
|
||||||
|
"market": offer["market"],
|
||||||
|
"currency": offer["currency"],
|
||||||
|
"condition": offer["condition"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for price in FIXTURE["prices"]:
|
||||||
|
offer = next(item for item in FIXTURE["offers"] if item["external_id"] == price["external_offer_id"])
|
||||||
|
_ok(
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/offers/{offer['id']}/prices",
|
||||||
|
json={
|
||||||
|
"id": price["id"],
|
||||||
|
"source_id": "manual",
|
||||||
|
"external_observation_id": price["external_observation_id"],
|
||||||
|
"observed_at": price["observed_at"],
|
||||||
|
"item_price": price["item_price"],
|
||||||
|
"shipping_price": price["shipping_price"],
|
||||||
|
"currency": price["currency"],
|
||||||
|
"availability": price["availability"],
|
||||||
|
"raw_payload": price["raw_payload"],
|
||||||
|
"confidence": price["confidence"],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
allowed=(200, 201),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_cases_and_replay(client: TestClient) -> None:
|
||||||
|
_seed(client)
|
||||||
|
cases = {
|
||||||
|
"A": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-333333333333"),
|
||||||
|
"B": ("22222222-2222-4222-8222-22222222222b", "33333333-3333-4333-8333-33333333333b"),
|
||||||
|
"C": ("22222222-2222-4222-8222-22222222222c", "33333333-3333-4333-8333-33333333333c"),
|
||||||
|
"D": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-33333333333d"),
|
||||||
|
}
|
||||||
|
created = {}
|
||||||
|
for label, (variant_id, offer_id) in cases.items():
|
||||||
|
response = client.post(
|
||||||
|
f"/api/v1/evaluate/{variant_id}",
|
||||||
|
json={
|
||||||
|
"profile_id": "mobile-development-notebook",
|
||||||
|
"profile_version": 1,
|
||||||
|
"offer_id": offer_id,
|
||||||
|
"as_of": AS_OF,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert response.status_code == 201, response.text
|
||||||
|
created[label] = response.json()
|
||||||
|
|
||||||
|
assert created["A"]["product_score"] == 90.21
|
||||||
|
assert created["A"]["price_score"] == 98.24
|
||||||
|
assert created["A"]["deal_score"] == 93.42
|
||||||
|
assert created["A"]["classification"] == "EXCEPTIONAL"
|
||||||
|
assert created["A"]["eligible"] is True
|
||||||
|
assert created["A"]["price_reference_scope"] == "offer"
|
||||||
|
|
||||||
|
assert created["B"]["product_eligibility"] == "INELIGIBLE"
|
||||||
|
assert created["B"]["offer_eligibility"] == "ELIGIBLE"
|
||||||
|
assert created["B"]["deal_score"] is None
|
||||||
|
assert created["B"]["product_score"] is not None
|
||||||
|
|
||||||
|
assert created["C"]["product_eligibility"] == "UNKNOWN"
|
||||||
|
assert created["C"]["classification"] == "INSUFFICIENT_DATA"
|
||||||
|
|
||||||
|
assert created["D"]["product_eligibility"] == "ELIGIBLE"
|
||||||
|
assert created["D"]["offer_eligibility"] == "INELIGIBLE"
|
||||||
|
assert created["D"]["product_score"] == 90.21
|
||||||
|
assert created["D"]["price_score"] is not None
|
||||||
|
assert created["D"]["deal_score"] is None
|
||||||
|
|
||||||
|
stored = client.get(f"/api/v1/evaluations/{created['A']['id']}")
|
||||||
|
assert stored.status_code == 200
|
||||||
|
assert stored.json()["deal_score"] == created["A"]["deal_score"]
|
||||||
|
assert stored.json()["input_snapshot"] == created["A"]["input_snapshot"]
|
||||||
|
|
||||||
|
extra = client.post(
|
||||||
|
"/api/v1/offers/33333333-3333-4333-8333-333333333333/prices",
|
||||||
|
json={
|
||||||
|
"source_id": "manual",
|
||||||
|
"external_observation_id": "a-later",
|
||||||
|
"observed_at": "2026-10-03T11:00:00Z",
|
||||||
|
"item_price": "999.00",
|
||||||
|
"shipping_price": "0.00",
|
||||||
|
"currency": "EUR",
|
||||||
|
"availability": "in_stock",
|
||||||
|
"raw_payload": {"total": "999.00"},
|
||||||
|
"confidence": 1.0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert extra.status_code == 201
|
||||||
|
replay = client.get(f"/api/v1/evaluations/{created['A']['id']}")
|
||||||
|
assert replay.json()["deal_score"] == created["A"]["deal_score"]
|
||||||
|
assert replay.json()["price_statistics"]["current_total"] == created["A"]["price_statistics"]["current_total"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_contracts(client: TestClient) -> None:
|
||||||
|
missing = client.get("/api/v1/products/00000000-0000-4000-8000-000000000000")
|
||||||
|
assert missing.status_code == 404
|
||||||
|
assert "error" in missing.json()
|
||||||
|
assert missing.json()["error"]["code"] == "NOT_FOUND"
|
||||||
|
|
||||||
|
bad = client.post("/api/v1/products", json={"category": "notebook"})
|
||||||
|
assert bad.status_code == 422
|
||||||
|
assert "error" in bad.json()
|
||||||
|
|
||||||
|
client.post("/api/v1/attribute-definitions", json={"key": "ports", "value_type": "string_list"})
|
||||||
|
again = client.post("/api/v1/attribute-definitions", json={"key": "ports", "value_type": "string_list"})
|
||||||
|
assert again.status_code == 409
|
||||||
|
|
||||||
|
no_version = client.get("/api/v1/profiles/mobile-development-notebook")
|
||||||
|
assert no_version.status_code == 422
|
||||||
0
backend/tests/integration/__init__.py
Normal file
0
backend/tests/integration/__init__.py
Normal file
249
backend/tests/integration/test_discovery_lifecycle.py
Normal file
249
backend/tests/integration/test_discovery_lifecycle.py
Normal file
|
|
@ -0,0 +1,249 @@
|
||||||
|
from copy import deepcopy
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.discovery.enums import DiffStatus, LifecycleStatus, MatchStatus, RunStatus
|
||||||
|
from app.errors import AppError
|
||||||
|
from app.persistence.db import SessionLocal, engine
|
||||||
|
from app.persistence.discovery_models import (
|
||||||
|
ProductObservationRecord,
|
||||||
|
ProjectCandidateRecord,
|
||||||
|
SourceProductReferenceRecord,
|
||||||
|
)
|
||||||
|
from app.services.discovery_service import (
|
||||||
|
CriterionMappingService,
|
||||||
|
DiscoverySearchService,
|
||||||
|
SearchExecutionService,
|
||||||
|
TaxonomyService,
|
||||||
|
)
|
||||||
|
from app.services.ingestion_service import IngestionService
|
||||||
|
from app.services.project_service import ProjectService
|
||||||
|
from app.sources.geizhals.adapter import GeizhalsAdapter
|
||||||
|
from scripts.bootstrap import run_bootstrap
|
||||||
|
from scripts.seed_notebook_pilot import seed_notebook_pilot
|
||||||
|
from tests.e2e.test_http_walkthrough import DEFINITIONS, PROFILE
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
ITEM_A = {
|
||||||
|
"source_external_id": "gh-nb-a",
|
||||||
|
"source_url": "https://geizhals.de/example-mobile-13-a.html",
|
||||||
|
"source_title": "Example Mobile 13 32GB",
|
||||||
|
"manufacturer": "Example",
|
||||||
|
"model": "Mobile 13",
|
||||||
|
"manufacturer_sku": "EM13-32-1T",
|
||||||
|
"gtin_ean": "4000000000001",
|
||||||
|
"best_price": "1298.00",
|
||||||
|
"currency": "EUR",
|
||||||
|
"availability": "in_stock",
|
||||||
|
"seller_count": 8,
|
||||||
|
"raw_attributes": {"RAM": "32GB"},
|
||||||
|
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "weight_kg": 1.2},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _item(suffix: str, **overrides) -> dict:
|
||||||
|
item = deepcopy(ITEM_A)
|
||||||
|
item["source_external_id"] = f"gh-nb-{suffix}"
|
||||||
|
item["source_url"] = f"https://geizhals.de/example-mobile-13-{suffix}.html"
|
||||||
|
item["manufacturer_sku"] = f"EM13-{suffix}"
|
||||||
|
item["gtin_ean"] = None
|
||||||
|
item["source_title"] = f"Example Mobile 13 {suffix}"
|
||||||
|
item.update(overrides)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
class ScriptedAdapter:
|
||||||
|
source_key = "geizhals"
|
||||||
|
|
||||||
|
def __init__(self, pages: list[dict]) -> None:
|
||||||
|
self.pages = pages
|
||||||
|
self._inner = GeizhalsAdapter()
|
||||||
|
|
||||||
|
def discover_taxonomy(self, category_ref: str) -> dict:
|
||||||
|
return self._inner.discover_taxonomy(category_ref)
|
||||||
|
|
||||||
|
def build_query(self, criteria, mappings) -> dict:
|
||||||
|
return self._inner.build_query(criteria, mappings)
|
||||||
|
|
||||||
|
def fetch_results(self, query, pagination) -> dict:
|
||||||
|
page = int(pagination.get("page") or 1)
|
||||||
|
return self.pages[page - 1]
|
||||||
|
|
||||||
|
def parse_results(self, response) -> list:
|
||||||
|
return list(response.get("items") or [])
|
||||||
|
|
||||||
|
def normalize_result(self, result) -> dict:
|
||||||
|
return self._inner.normalize_result(result)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def require_postgres() -> None:
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
connection.execute(text("SELECT 1"))
|
||||||
|
except Exception as exc:
|
||||||
|
pytest.skip(f"PostgreSQL not available: {exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def _session():
|
||||||
|
session = SessionLocal()
|
||||||
|
run_bootstrap(session)
|
||||||
|
seed_notebook_pilot(session)
|
||||||
|
ingestion = IngestionService(session)
|
||||||
|
for definition in DEFINITIONS:
|
||||||
|
try:
|
||||||
|
ingestion.create_definition(definition)
|
||||||
|
except AppError as exc:
|
||||||
|
if exc.status_code != 409:
|
||||||
|
raise
|
||||||
|
try:
|
||||||
|
ingestion.create_profile(PROFILE)
|
||||||
|
except AppError as exc:
|
||||||
|
if exc.status_code != 409:
|
||||||
|
raise
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_search(session, adapter, threshold=300):
|
||||||
|
taxonomy = TaxonomyService(session, adapter).discover("geizhals", "nb")
|
||||||
|
mappings = CriterionMappingService(session)
|
||||||
|
mappings.create(
|
||||||
|
{
|
||||||
|
"criterion_key": "ram_gb",
|
||||||
|
"source_id": "geizhals",
|
||||||
|
"source_category_id": taxonomy["id"],
|
||||||
|
"external_filter_key": "17023",
|
||||||
|
"source_value": "ab 24GB",
|
||||||
|
"encoding": {"url_param": "xf"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
project = ProjectService(session).create(
|
||||||
|
{
|
||||||
|
"name": "Lifecycle",
|
||||||
|
"profile_id": "mobile-development-notebook",
|
||||||
|
"profile_version": 1,
|
||||||
|
"criteria": {"items": [{"id": "ram", "criterion": "ram_gb", "operator": ">=", "value": 24}]},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
search = DiscoverySearchService(session, adapter).create(
|
||||||
|
{
|
||||||
|
"buying_project_id": project["id"],
|
||||||
|
"source_id": "geizhals",
|
||||||
|
"source_category_id": taxonomy["id"],
|
||||||
|
"broad_result_threshold": threshold,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return search["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def _page(items, status=200, page_count=1):
|
||||||
|
return {"page": 1, "page_count": page_count, "result_count": len(items), "http_status": status, "items": items}
|
||||||
|
|
||||||
|
|
||||||
|
def test_partial_does_not_mark_missing_or_disappeared():
|
||||||
|
session = _session()
|
||||||
|
try:
|
||||||
|
adapter = ScriptedAdapter([_page([_item("partial")])])
|
||||||
|
search_id = _prepare_search(session, adapter)
|
||||||
|
first = SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
assert first["status"] == RunStatus.SUCCESS.value
|
||||||
|
ref_id = first["items"][0]["source_product_reference_id"]
|
||||||
|
adapter.pages = [_page([], status=429)]
|
||||||
|
second = SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
assert second["status"] == RunStatus.PARTIAL.value
|
||||||
|
assert all(item["diff_status"] != DiffStatus.MISSING.value for item in second["items"])
|
||||||
|
ref = session.get(SourceProductReferenceRecord, ref_id)
|
||||||
|
assert ref is not None
|
||||||
|
assert ref.lifecycle_status == LifecycleStatus.ACTIVE.value
|
||||||
|
assert ref.consecutive_missing_success_runs == 0
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_three_success_misses_disappear_and_reappear():
|
||||||
|
session = _session()
|
||||||
|
try:
|
||||||
|
item = _item("gone")
|
||||||
|
adapter = ScriptedAdapter([_page([item])])
|
||||||
|
search_id = _prepare_search(session, adapter)
|
||||||
|
first = SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
ref_id = first["items"][0]["source_product_reference_id"]
|
||||||
|
adapter.pages = [_page([])]
|
||||||
|
for _ in range(3):
|
||||||
|
SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
ref = session.get(SourceProductReferenceRecord, ref_id)
|
||||||
|
assert ref is not None
|
||||||
|
assert ref.lifecycle_status == LifecycleStatus.DISAPPEARED.value
|
||||||
|
changed = deepcopy(item)
|
||||||
|
changed["best_price"] = "999.00"
|
||||||
|
adapter.pages = [_page([changed])]
|
||||||
|
run = SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
session.refresh(ref)
|
||||||
|
assert ref.lifecycle_status == LifecycleStatus.ACTIVE.value
|
||||||
|
assert any(item["diff_status"] == DiffStatus.CHANGED.value for item in run["items"])
|
||||||
|
assert all(item["diff_status"] != DiffStatus.NEW.value for item in run["items"])
|
||||||
|
prices = [
|
||||||
|
row.best_price
|
||||||
|
for row in session.query(ProductObservationRecord).filter(
|
||||||
|
ProductObservationRecord.source_product_reference_id == ref_id
|
||||||
|
)
|
||||||
|
]
|
||||||
|
assert Decimal("1298.00") in prices
|
||||||
|
assert Decimal("999.00") in prices
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_uncertain_match_stays_separate():
|
||||||
|
session = _session()
|
||||||
|
try:
|
||||||
|
first = _item("matched", manufacturer="Example", model="Uncertain 13")
|
||||||
|
second = _item(
|
||||||
|
"uncertain",
|
||||||
|
source_external_id=None,
|
||||||
|
manufacturer_sku=None,
|
||||||
|
source_url="https://geizhals.de/other-uncertain.html",
|
||||||
|
source_title=first["source_title"],
|
||||||
|
manufacturer="Example",
|
||||||
|
model="Uncertain 13",
|
||||||
|
normalized_attributes={"ram_gb": 16, "storage_gb": 512, "weight_kg": 2.0},
|
||||||
|
)
|
||||||
|
adapter = ScriptedAdapter([_page([first, second])])
|
||||||
|
search_id = _prepare_search(session, adapter)
|
||||||
|
project_id = DiscoverySearchService(session, adapter).get(search_id)["buying_project_id"]
|
||||||
|
run = SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
assert run["status"] == RunStatus.SUCCESS.value
|
||||||
|
ref_ids = {item["source_product_reference_id"] for item in run["items"]}
|
||||||
|
assert len(ref_ids) == 2
|
||||||
|
refs = [session.get(SourceProductReferenceRecord, ref_id) for ref_id in ref_ids]
|
||||||
|
assert MatchStatus.MATCH_UNCERTAIN.value in {ref.match_status for ref in refs if ref is not None}
|
||||||
|
candidates = (
|
||||||
|
session.query(ProjectCandidateRecord)
|
||||||
|
.filter(ProjectCandidateRecord.buying_project_id == project_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
assert len(candidates) == 2
|
||||||
|
assert MatchStatus.MATCH_UNCERTAIN.value in {row.match_status for row in candidates}
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_backoff_and_blocked():
|
||||||
|
session = _session()
|
||||||
|
try:
|
||||||
|
adapter = ScriptedAdapter([_page([ITEM_A], status=429)])
|
||||||
|
search_id = _prepare_search(session, adapter)
|
||||||
|
backoff = SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
assert backoff["status"] == RunStatus.PARTIAL.value
|
||||||
|
assert backoff["error_code"] == "BACKOFF"
|
||||||
|
adapter.pages = [_page([ITEM_A], status=403)]
|
||||||
|
blocked = SearchExecutionService(session, adapter).execute(search_id)
|
||||||
|
assert blocked["status"] == RunStatus.BLOCKED.value
|
||||||
|
stored = DiscoverySearchService(session, adapter).get(search_id)
|
||||||
|
assert stored["status"] == "BLOCKED"
|
||||||
|
assert stored["next_run_at"] is None
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user