Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 246a61ece5 | |||
| d280575aca |
|
|
@ -1,6 +1,6 @@
|
||||||
# Copy to .env next to docker-compose*.yml. Never commit .env.
|
# Copy to .env next to docker-compose*.yml. Never commit .env.
|
||||||
# Dev server: ~/docker/buyingagent-dev/.env
|
# Dev server: /home/lars/docker/buyingagent-dev/.env
|
||||||
# Prod server: ~/docker/buyingagent/.env
|
# Prod server: /home/lars/docker/buyingagent/.env
|
||||||
|
|
||||||
# ─── local / production (docker-compose.yml) ────────────────────────────────
|
# ─── local / production (docker-compose.yml) ────────────────────────────────
|
||||||
DB_NAME=buyingagent
|
DB_NAME=buyingagent
|
||||||
|
|
@ -10,9 +10,6 @@ ENVIRONMENT=production
|
||||||
ENGINE_BUILD=dev
|
ENGINE_BUILD=dev
|
||||||
BUYING_AGENT_API_BIND=127.0.0.1
|
BUYING_AGENT_API_BIND=127.0.0.1
|
||||||
BUYING_AGENT_API_PORT=8000
|
BUYING_AGENT_API_PORT=8000
|
||||||
BUYING_AGENT_FRONTEND_BIND=127.0.0.1
|
|
||||||
BUYING_AGENT_FRONTEND_PORT=4173
|
|
||||||
CORS_ALLOWED_ORIGINS=http://localhost:4173,http://127.0.0.1:4173,http://localhost:5173,http://127.0.0.1:5173
|
|
||||||
|
|
||||||
# Production on the Raspberry Pi (192.168.2.49):
|
# Production on the Raspberry Pi (192.168.2.49):
|
||||||
# BUYING_AGENT_API_BIND=0.0.0.0
|
# BUYING_AGENT_API_BIND=0.0.0.0
|
||||||
|
|
|
||||||
1
.gitattributes
vendored
1
.gitattributes
vendored
|
|
@ -1 +0,0 @@
|
||||||
*.sh text eol=lf
|
|
||||||
|
|
@ -14,7 +14,7 @@ jobs:
|
||||||
set -e
|
set -e
|
||||||
echo "=== Deploying BuyingAgent to DEVELOPMENT ==="
|
echo "=== Deploying BuyingAgent to DEVELOPMENT ==="
|
||||||
REPO="http://192.168.2.144:3000/Lars/BuyingAgent.git"
|
REPO="http://192.168.2.144:3000/Lars/BuyingAgent.git"
|
||||||
TARGET="$HOME/docker/buyingagent-dev"
|
TARGET="/home/lars/docker/buyingagent-dev"
|
||||||
mkdir -p "$TARGET"
|
mkdir -p "$TARGET"
|
||||||
cd "$TARGET"
|
cd "$TARGET"
|
||||||
if [ ! -d .git ]; then
|
if [ ! -d .git ]; then
|
||||||
|
|
@ -41,13 +41,7 @@ jobs:
|
||||||
export ENGINE_BUILD BUYING_AGENT_API_PORT
|
export ENGINE_BUILD BUYING_AGENT_API_PORT
|
||||||
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
||||||
BUYING_AGENT_API_PORT=8095
|
BUYING_AGENT_API_PORT=8095
|
||||||
if ! grep -q '^CORS_ALLOWED_ORIGINS=' .env; then
|
docker compose -f docker-compose.dev-env.yml build --no-cache api
|
||||||
echo 'CORS_ALLOWED_ORIGINS=http://192.168.2.49:5174,http://localhost:5173' >> .env
|
|
||||||
fi
|
|
||||||
if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then
|
|
||||||
echo 'BUYING_AGENT_FRONTEND_PORT=5174' >> .env
|
|
||||||
fi
|
|
||||||
COMPOSE_BAKE=0 docker compose -f docker-compose.dev-env.yml build --pull=false api frontend
|
|
||||||
if ! ENGINE_BUILD="$ENGINE_BUILD" BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
if ! ENGINE_BUILD="$ENGINE_BUILD" BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
||||||
docker compose -f docker-compose.dev-env.yml up -d --wait; then
|
docker compose -f docker-compose.dev-env.yml up -d --wait; then
|
||||||
echo "compose up --wait failed — api logs:"
|
echo "compose up --wait failed — api logs:"
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ jobs:
|
||||||
set -e
|
set -e
|
||||||
echo "=== Deploying BuyingAgent to PRODUCTION ==="
|
echo "=== Deploying BuyingAgent to PRODUCTION ==="
|
||||||
REPO="http://192.168.2.144:3000/Lars/BuyingAgent.git"
|
REPO="http://192.168.2.144:3000/Lars/BuyingAgent.git"
|
||||||
TARGET="$HOME/docker/buyingagent"
|
TARGET="/home/lars/docker/buyingagent"
|
||||||
mkdir -p "$TARGET"
|
mkdir -p "$TARGET"
|
||||||
cd "$TARGET"
|
cd "$TARGET"
|
||||||
if [ ! -d .git ]; then
|
if [ ! -d .git ]; then
|
||||||
|
|
@ -48,13 +48,7 @@ jobs:
|
||||||
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
||||||
BUYING_AGENT_API_BIND=0.0.0.0
|
BUYING_AGENT_API_BIND=0.0.0.0
|
||||||
BUYING_AGENT_API_PORT=8006
|
BUYING_AGENT_API_PORT=8006
|
||||||
if ! grep -q '^CORS_ALLOWED_ORIGINS=' .env; then
|
docker compose build --no-cache api
|
||||||
echo 'CORS_ALLOWED_ORIGINS=http://192.168.2.49:4173,http://localhost:4173' >> .env
|
|
||||||
fi
|
|
||||||
if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then
|
|
||||||
echo 'BUYING_AGENT_FRONTEND_PORT=4173' >> .env
|
|
||||||
fi
|
|
||||||
COMPOSE_BAKE=0 docker compose build --pull=false api frontend
|
|
||||||
if ! ENGINE_BUILD="$ENGINE_BUILD" \
|
if ! ENGINE_BUILD="$ENGINE_BUILD" \
|
||||||
BUYING_AGENT_API_BIND="$BUYING_AGENT_API_BIND" \
|
BUYING_AGENT_API_BIND="$BUYING_AGENT_API_BIND" \
|
||||||
BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
||||||
|
|
|
||||||
|
|
@ -9,30 +9,6 @@ on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
pytest-backend:
|
|
||||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: pytest against Dev Postgres (buyingagent_test)
|
|
||||||
run: |
|
|
||||||
set -e
|
|
||||||
APP_DIR="$HOME/docker/buyingagent-dev"
|
|
||||||
echo "pytest uses Dev Postgres (${APP_DIR}); event=${{ github.event_name }}"
|
|
||||||
cd "$APP_DIR"
|
|
||||||
for i in $(seq 1 60); do
|
|
||||||
if [ -f scripts/ci-pytest.sh ]; then
|
|
||||||
echo "ci-pytest.sh ready (attempt $i)"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
if [ "$i" -eq 60 ]; then
|
|
||||||
echo "timeout waiting for scripts/ci-pytest.sh after Dev deploy"
|
|
||||||
ls -la scripts || true
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
sh scripts/ci-pytest.sh
|
|
||||||
|
|
||||||
smoke-dev:
|
smoke-dev:
|
||||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
@ -40,7 +16,7 @@ jobs:
|
||||||
- name: Non-destructive smoke against the running Dev API
|
- name: Non-destructive smoke against the running Dev API
|
||||||
run: |
|
run: |
|
||||||
set -e
|
set -e
|
||||||
echo "live Dev API (buyingagent-dev), read-only health"
|
echo "wait for live Dev API (buyingagent-dev)"
|
||||||
for i in $(seq 1 60); do
|
for i in $(seq 1 60); do
|
||||||
if curl -sf http://localhost:8095/health/live >/dev/null; then
|
if curl -sf http://localhost:8095/health/live >/dev/null; then
|
||||||
echo "DEV /health/live ready (attempt $i)"
|
echo "DEV /health/live ready (attempt $i)"
|
||||||
|
|
@ -57,18 +33,3 @@ jobs:
|
||||||
curl -sf http://localhost:8095/health/ready
|
curl -sf http://localhost:8095/health/ready
|
||||||
echo
|
echo
|
||||||
echo "LAN smoke OK: http://192.168.2.49:8095"
|
echo "LAN smoke OK: http://192.168.2.49:8095"
|
||||||
|
|
||||||
frontend-checks:
|
|
||||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Frontend typecheck, lint, test, build
|
|
||||||
run: |
|
|
||||||
set -e
|
|
||||||
APP_DIR="$HOME/docker/buyingagent-dev"
|
|
||||||
cd "$APP_DIR/frontend"
|
|
||||||
npm ci || npm install
|
|
||||||
npm run typecheck
|
|
||||||
npm run lint
|
|
||||||
npm test
|
|
||||||
npm run build
|
|
||||||
|
|
|
||||||
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -14,10 +14,6 @@
|
||||||
dist/
|
dist/
|
||||||
build/
|
build/
|
||||||
|
|
||||||
# Frontend
|
|
||||||
node_modules/
|
|
||||||
frontend/dev-dist/
|
|
||||||
|
|
||||||
# OS / editor
|
# OS / editor
|
||||||
.DS_Store
|
.DS_Store
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
|
|
|
||||||
|
|
@ -1,377 +0,0 @@
|
||||||
# Buying Agent – Architecture Addendum v1
|
|
||||||
|
|
||||||
**Status:** ergänzende Architekturvorgabe
|
|
||||||
**Gültig zusammen mit:** `buying-agent-v0.1-spec-v1.1.md`
|
|
||||||
**Stand:** 2026-10-03
|
|
||||||
|
|
||||||
## 1. Zweck
|
|
||||||
|
|
||||||
Dieses Addendum ergänzt die bestehende Spezifikation um übergreifende Architekturentscheidungen für das spätere Zielsystem.
|
|
||||||
|
|
||||||
Es ersetzt **nicht** die fachliche Spezifikation `buying-agent-v0.1-spec-v1.1.md`.
|
|
||||||
|
|
||||||
Bei Konflikten zwischen beiden Dokumenten hat dieses Addendum für die hier behandelten Architekturthemen Vorrang.
|
|
||||||
|
|
||||||
Ziel ist, den aktuellen v0.1-Slice schlank zu halten, ohne spätere Erweiterungen auf Frontend, mehrere Kaufprojekte und mehrere Benutzer unnötig zu erschweren.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. Zielstruktur des Repositories
|
|
||||||
|
|
||||||
Das Projekt wird als Monorepo strukturiert:
|
|
||||||
|
|
||||||
```text
|
|
||||||
buying-agent/
|
|
||||||
├── README.md
|
|
||||||
├── docker-compose.yml
|
|
||||||
├── .env.example
|
|
||||||
├── .gitignore
|
|
||||||
├── docs/
|
|
||||||
│ ├── buying-agent-v0.1-spec-v1.1.md
|
|
||||||
│ └── buying-agent-architecture-addendum-v1.md
|
|
||||||
├── backend/
|
|
||||||
│ ├── Dockerfile
|
|
||||||
│ ├── pyproject.toml
|
|
||||||
│ ├── uv.lock
|
|
||||||
│ ├── alembic.ini
|
|
||||||
│ ├── migrations/
|
|
||||||
│ ├── app/
|
|
||||||
│ ├── rules/
|
|
||||||
│ ├── profiles/
|
|
||||||
│ ├── fixtures/
|
|
||||||
│ ├── scripts/
|
|
||||||
│ └── tests/
|
|
||||||
└── frontend/
|
|
||||||
└── README.md
|
|
||||||
```
|
|
||||||
|
|
||||||
### Regeln
|
|
||||||
|
|
||||||
- `docker-compose.yml` bleibt im Repository-Root.
|
|
||||||
- Das Backend ist der aktuell implementierte v0.1-Scope.
|
|
||||||
- `frontend/` wird in v0.1 **nicht implementiert**.
|
|
||||||
- `frontend/README.md` dokumentiert lediglich, dass dort später die Benutzeroberfläche entsteht.
|
|
||||||
- Keine zusätzliche `infra/`-Struktur in v0.1, solange sie nicht konkret benötigt wird.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. Backend-/Frontend-Grenze
|
|
||||||
|
|
||||||
Das Backend bleibt das **System of Record** und besitzt:
|
|
||||||
|
|
||||||
- Domain-Modell
|
|
||||||
- Constraints
|
|
||||||
- Scoring
|
|
||||||
- Evaluation
|
|
||||||
- Persistenz
|
|
||||||
- Preis- und Produktlogik
|
|
||||||
- zukünftige Kaufentscheidungslogik
|
|
||||||
|
|
||||||
Das spätere Frontend konsumiert ausschließlich die versionierte Backend-API, primär unter:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/api/v1
|
|
||||||
```
|
|
||||||
|
|
||||||
Das Frontend enthält keine fachlich verbindliche Business-Logik.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 4. Zukünftige Multi-User-Architektur
|
|
||||||
|
|
||||||
Das Zielsystem soll später von Familie und gegebenenfalls engen Freunden genutzt werden können.
|
|
||||||
|
|
||||||
Die Architektur darf daher nicht von einem impliziten Single-User-Modell ausgehen.
|
|
||||||
|
|
||||||
Zielmodell:
|
|
||||||
|
|
||||||
```text
|
|
||||||
User
|
|
||||||
↕
|
|
||||||
WorkspaceMember
|
|
||||||
↕
|
|
||||||
Workspace
|
|
||||||
↓
|
|
||||||
BuyingProject
|
|
||||||
↓
|
|
||||||
ProjectCandidate
|
|
||||||
↓
|
|
||||||
ProductVariant
|
|
||||||
```
|
|
||||||
|
|
||||||
Ein Benutzer kann Mitglied in mehreren Workspaces sein.
|
|
||||||
|
|
||||||
Beispiele:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Lars
|
|
||||||
├── Privat
|
|
||||||
├── Familie
|
|
||||||
└── Freunde
|
|
||||||
```
|
|
||||||
|
|
||||||
### Noch nicht Bestandteil von v0.1
|
|
||||||
|
|
||||||
Nicht implementieren:
|
|
||||||
|
|
||||||
- Login
|
|
||||||
- Authentifizierung
|
|
||||||
- Passwortverwaltung
|
|
||||||
- Einladungen
|
|
||||||
- RBAC
|
|
||||||
- komplexe ACLs
|
|
||||||
- Sharing-UI
|
|
||||||
- Workspace-Administration
|
|
||||||
|
|
||||||
Die aktuelle Architektur darf deren spätere Einführung aber nicht unnötig erschweren.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. Globale versus Workspace-bezogene Daten
|
|
||||||
|
|
||||||
### Global wiederverwendbare Daten
|
|
||||||
|
|
||||||
Diese Entitäten bleiben systemweit gemeinsam nutzbar:
|
|
||||||
|
|
||||||
- Product
|
|
||||||
- ProductVariant
|
|
||||||
- Offer
|
|
||||||
- PriceObservation
|
|
||||||
- AttributeDefinition
|
|
||||||
- Source
|
|
||||||
|
|
||||||
Sie erhalten **keine** `user_id` oder `workspace_id`.
|
|
||||||
|
|
||||||
Ein Produkt oder Angebot darf nicht pro Benutzer oder Workspace dupliziert werden.
|
|
||||||
|
|
||||||
### Workspace-bezogene Daten
|
|
||||||
|
|
||||||
Spätere benutzerbezogene Beschaffungskontexte gehören zu einem Workspace:
|
|
||||||
|
|
||||||
- BuyingProject
|
|
||||||
- ProjectCandidate
|
|
||||||
- projektspezifische Profil-/Anforderungsauswahl
|
|
||||||
- Notification Preferences
|
|
||||||
- Beschaffungsdeadline
|
|
||||||
- Zielpreis
|
|
||||||
- Kaufentscheidungsstatus
|
|
||||||
- gegebenenfalls spätere Warenkorb-/Beschaffungsinformationen
|
|
||||||
|
|
||||||
Diese Objekte werden erst in späteren Slices implementiert.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 6. BuyingProject als zukünftiges Kernobjekt
|
|
||||||
|
|
||||||
Ein `BuyingProject` repräsentiert ein konkretes Beschaffungsvorhaben.
|
|
||||||
|
|
||||||
Beispiele:
|
|
||||||
|
|
||||||
- Mobiler Entwicklungsrechner
|
|
||||||
- Lokaler LLM-Server
|
|
||||||
- Fernseher
|
|
||||||
- Kamera
|
|
||||||
|
|
||||||
Ein Workspace kann mehrere BuyingProjects parallel besitzen.
|
|
||||||
|
|
||||||
Ein BuyingProject kann mehrere ProjectCandidates beobachten.
|
|
||||||
|
|
||||||
```text
|
|
||||||
BuyingProject
|
|
||||||
├── Candidate A → ProductVariant
|
|
||||||
├── Candidate B → ProductVariant
|
|
||||||
└── Candidate C → ProductVariant
|
|
||||||
```
|
|
||||||
|
|
||||||
### Wichtig für v0.1
|
|
||||||
|
|
||||||
`BuyingProject` und `ProjectCandidate` müssen in v0.1 noch nicht implementiert werden.
|
|
||||||
|
|
||||||
Die aktuelle Implementierung darf jedoch nicht annehmen:
|
|
||||||
|
|
||||||
- es existiert genau ein aktives Profil,
|
|
||||||
- es existiert genau ein Kaufvorhaben,
|
|
||||||
- es gibt globale Singleton-Konfiguration für einen aktuellen Kauf,
|
|
||||||
- alle Daten gehören automatisch demselben Benutzer,
|
|
||||||
- es existiert nur eine gleichzeitig relevante Evaluation.
|
|
||||||
|
|
||||||
Mehrere unabhängige Profile und Evaluationen müssen bereits parallel koexistieren können.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 7. Rolle von n8n
|
|
||||||
|
|
||||||
n8n bleibt eine externe Orchestrierungsschicht.
|
|
||||||
|
|
||||||
Spätere mögliche Aufgaben:
|
|
||||||
|
|
||||||
- Scheduler
|
|
||||||
- Preisabfragen
|
|
||||||
- Datenimport
|
|
||||||
- Benachrichtigungen
|
|
||||||
- externe Integrationen
|
|
||||||
- volatile oder häufig veränderte Workflows
|
|
||||||
|
|
||||||
Kern-Business-Logik bleibt im Programmcode, insbesondere wenn sie:
|
|
||||||
|
|
||||||
- deterministisch sein soll,
|
|
||||||
- versioniert werden muss,
|
|
||||||
- getestet werden muss,
|
|
||||||
- fachliche Entscheidungen reproduzierbar machen soll.
|
|
||||||
|
|
||||||
n8n darf für veränderliche Prozesslogik eingesetzt werden, aber nicht zur Voraussetzung für das Funktionieren der Buying Engine werden.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 8. Zukünftiges Frontend
|
|
||||||
|
|
||||||
Das Zielsystem soll später eine einfach bedienbare Oberfläche erhalten.
|
|
||||||
|
|
||||||
Die Standardoberfläche soll auch für nichttechnische Familienmitglieder verständlich bleiben.
|
|
||||||
|
|
||||||
Die Engine darf gleichzeitig deutlich leistungsfähiger sein als die Oberfläche.
|
|
||||||
|
|
||||||
Voraussichtliche spätere Kernbereiche:
|
|
||||||
|
|
||||||
- Übersicht laufender Kaufprojekte
|
|
||||||
- Kandidaten je Projekt
|
|
||||||
- aktueller bester Deal
|
|
||||||
- Preisverlauf
|
|
||||||
- Produkt-Fit
|
|
||||||
- Offer-Fit
|
|
||||||
- DealScore
|
|
||||||
- verständliche Gründe
|
|
||||||
- Beobachtungsstatus
|
|
||||||
- konkrete Kaufempfehlung
|
|
||||||
- ggf. spätere Benachrichtigungs- und Warenkorbfunktionen
|
|
||||||
|
|
||||||
Keine dieser UI-Funktionen ist Bestandteil von v0.1.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 9. Kaufentscheidungslogik
|
|
||||||
|
|
||||||
Langfristig soll das System nicht nur Scores liefern, sondern auch konfigurierbare Kaufentscheidungen unterstützen.
|
|
||||||
|
|
||||||
Beispiele möglicher Einflussfaktoren:
|
|
||||||
|
|
||||||
- Zielpreis
|
|
||||||
- Maximalpreis
|
|
||||||
- spätestes Beschaffungsdatum
|
|
||||||
- aktueller Preis
|
|
||||||
- Preisentwicklung der letzten Tage/Wochen/Monate
|
|
||||||
- Abstand zu historischem Median
|
|
||||||
- Produkt-Fit
|
|
||||||
- Offer-Fit
|
|
||||||
- Verfügbarkeit
|
|
||||||
- Lieferzeit
|
|
||||||
- Händlerqualität
|
|
||||||
- alternative Kandidaten
|
|
||||||
- Beschaffungsdringlichkeit
|
|
||||||
|
|
||||||
Die Kaufentscheidung soll nachvollziehbar und möglichst deterministisch bleiben.
|
|
||||||
|
|
||||||
LLMs dürfen später Erklärungen, Normalisierung oder qualitative Interpretation unterstützen, sollen aber keine nicht nachvollziehbare Blackbox-Entscheidung erzwingen.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 10. Datenquellen
|
|
||||||
|
|
||||||
Das Zielsystem muss mehrere Produkt- und Preisquellen unterstützen können.
|
|
||||||
|
|
||||||
Mögliche spätere Quellen:
|
|
||||||
|
|
||||||
- Amazon / Keepa
|
|
||||||
- Preisvergleichsportale
|
|
||||||
- Hersteller
|
|
||||||
- Händler
|
|
||||||
- weitere APIs
|
|
||||||
|
|
||||||
v0.1 implementiert weiterhin **keine reale externe Quelle**.
|
|
||||||
|
|
||||||
Die bestehende Source-Adapter-Grenze bleibt die vorgesehene Erweiterungsstelle.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 11. Scope-Grenze für die laufende v0.1-Implementierung
|
|
||||||
|
|
||||||
Dieses Addendum darf nicht als Anlass dienen, v0.1 unnötig zu vergrößern.
|
|
||||||
|
|
||||||
Insbesondere **nicht jetzt implementieren**:
|
|
||||||
|
|
||||||
- echtes Frontend
|
|
||||||
- Authentifizierung
|
|
||||||
- User Management
|
|
||||||
- Workspace Management
|
|
||||||
- RBAC
|
|
||||||
- Einladungen
|
|
||||||
- BuyingProject
|
|
||||||
- ProjectCandidate
|
|
||||||
- Notifications
|
|
||||||
- n8n
|
|
||||||
- externe Preisquellen
|
|
||||||
- automatische Produktsuche
|
|
||||||
- Warenkorbmanagement
|
|
||||||
|
|
||||||
Ziel ist Architekturkompatibilität, nicht vorgezogene Feature-Entwicklung.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 12. Arbeitsauftrag für die laufende Implementierung
|
|
||||||
|
|
||||||
Der Coding-Agent soll vor Fortsetzung des laufenden Builds einen kurzen Architecture-Alignment-Check durchführen.
|
|
||||||
|
|
||||||
### Prüfen
|
|
||||||
|
|
||||||
1. Entspricht die Repository-Struktur bereits der Monorepo-Struktur mit:
|
|
||||||
- `backend/`
|
|
||||||
- `frontend/`
|
|
||||||
- `docs/`
|
|
||||||
- Root-`docker-compose.yml`
|
|
||||||
|
|
||||||
2. Gibt es Annahmen über:
|
|
||||||
- genau einen Benutzer,
|
|
||||||
- genau ein aktives Profil,
|
|
||||||
- genau ein Kaufprojekt,
|
|
||||||
- globale Singleton-Zustände?
|
|
||||||
|
|
||||||
3. Sind Product, ProductVariant, Offer und PriceObservation weiterhin global und unabhängig von späteren Workspaces modelliert?
|
|
||||||
|
|
||||||
4. Ist das Backend klar von zukünftiger Frontend- und n8n-Logik getrennt?
|
|
||||||
|
|
||||||
### Änderungen
|
|
||||||
|
|
||||||
Nur notwendige strukturelle Anpassungen durchführen.
|
|
||||||
|
|
||||||
Keine vorgezogene Multi-User- oder Frontend-Funktionalität implementieren.
|
|
||||||
|
|
||||||
Die fachlichen Anforderungen aus `buying-agent-v0.1-spec-v1.1.md` bleiben vollständig gültig.
|
|
||||||
|
|
||||||
### Abschlussbericht
|
|
||||||
|
|
||||||
Dokumentieren:
|
|
||||||
|
|
||||||
- geänderte oder verschobene Dateien,
|
|
||||||
- notwendige Architekturänderungen,
|
|
||||||
- Änderungen an Datenbank/Migrationen,
|
|
||||||
- ausgeführte Tests,
|
|
||||||
- Testergebnisse,
|
|
||||||
- verbleibende Risiken für spätere Workspace-/BuyingProject-Erweiterung.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 13. Vorrang und Weiterentwicklung
|
|
||||||
|
|
||||||
Dieses Dokument ist ein Architektur-Addendum zur v0.1-Spezifikation.
|
|
||||||
|
|
||||||
Nach Abschluss von v0.1 sollen die Erkenntnisse aus:
|
|
||||||
|
|
||||||
- `buying-agent-v0.1-spec-v1.1.md`
|
|
||||||
- diesem Addendum
|
|
||||||
- der realen Implementierung
|
|
||||||
|
|
||||||
in eine konsolidierte Folgespezifikation für den nächsten Slice übernommen werden.
|
|
||||||
|
|
||||||
Die Kernregel lautet:
|
|
||||||
|
|
||||||
> Nicht vorzeitig bauen, aber bekannte Zielanforderungen nicht architektonisch verbauen.
|
|
||||||
|
|
@ -1,987 +0,0 @@
|
||||||
# Buying Agent – Frontend MVP Specification v0.1
|
|
||||||
|
|
||||||
**Status:** implementierungsbereit
|
|
||||||
**Spezifikationsversion:** 0.1
|
|
||||||
**Stand:** 2026-10-04
|
|
||||||
|
|
||||||
**Ergänzt verbindlich:**
|
|
||||||
1. `buying-agent-v0.1-spec-v1.1.md`
|
|
||||||
2. `buying-agent-architecture-addendum-v1.md`
|
|
||||||
3. `buying-agent-source-discovery-spec-v0.1.md`
|
|
||||||
|
|
||||||
Diese Frontend-Spezifikation hebt für den nutzbaren MVP die frühere Aussage „kein Frontend in diesem Slice“ auf.
|
|
||||||
Sie ändert keine fachliche Backend-Logik; sie definiert die Benutzeroberfläche und die dafür notwendigen API-Verträge.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1. Ziel
|
|
||||||
|
|
||||||
Der Buying Agent soll als **nutzbare PWA** einen vollständigen vertikalen Ablauf unterstützen:
|
|
||||||
|
|
||||||
```text
|
|
||||||
BuyingProject anlegen
|
|
||||||
↓
|
|
||||||
Produktkategorie wählen
|
|
||||||
↓
|
|
||||||
dynamische Kriterien aus Source-Taxonomie auswählen
|
|
||||||
↓
|
|
||||||
Kriterien konfigurieren
|
|
||||||
↓
|
|
||||||
Geizhals-Suche generieren
|
|
||||||
↓
|
|
||||||
SearchRun starten / Status verfolgen
|
|
||||||
↓
|
|
||||||
Kandidaten sehen
|
|
||||||
↓
|
|
||||||
Ranking + Shortlist sehen
|
|
||||||
↓
|
|
||||||
Kandidat öffnen
|
|
||||||
↓
|
|
||||||
Status, Historie, bisherige Ergebnisse und Empfehlungen sehen
|
|
||||||
↓
|
|
||||||
Kriterien bei Bedarf anpassen
|
|
||||||
```
|
|
||||||
|
|
||||||
Der MVP muss diesen Flow über die Oberfläche vollständig nutzbar machen.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. Technologiestack
|
|
||||||
|
|
||||||
Frontend:
|
|
||||||
|
|
||||||
- React
|
|
||||||
- TypeScript
|
|
||||||
- Vite
|
|
||||||
- PWA
|
|
||||||
- TanStack Query für Server-State / Data Fetching
|
|
||||||
- React Router oder äquivalente schlanke Routing-Lösung
|
|
||||||
- schlanke komponentenbasierte UI
|
|
||||||
- kein Next.js
|
|
||||||
- kein SSR
|
|
||||||
- keine SEO-Anforderungen
|
|
||||||
|
|
||||||
Das Frontend liegt im bestehenden Monorepo:
|
|
||||||
|
|
||||||
```text
|
|
||||||
buying-agent/
|
|
||||||
├── backend/
|
|
||||||
├── frontend/
|
|
||||||
├── docs/
|
|
||||||
└── docker-compose.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Backend bleibt System of Record.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. Gestaltungsprinzipien
|
|
||||||
|
|
||||||
1. **Einfach für Standardnutzer, leistungsfähig im Hintergrund.**
|
|
||||||
2. Keine technische Datenbank-/Engine-Sicht in der Standardoberfläche.
|
|
||||||
3. Dynamische Kriterien statt fest verdrahteter Notebook-Felder.
|
|
||||||
4. Responsive von Anfang an.
|
|
||||||
5. Zwei bewusst gestaltete Hauptlayouts:
|
|
||||||
- Desktop
|
|
||||||
- Mobile
|
|
||||||
6. Nüchternes, klares Design.
|
|
||||||
7. Bedienbarkeit vor visueller Komplexität.
|
|
||||||
8. Admin-/Source-Themen von der normalen Nutzung trennen.
|
|
||||||
9. Keine Businesslogik im Frontend.
|
|
||||||
10. Frontend konsumiert ausschließlich die versionierte Backend-API.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 4. PWA und Responsive Layout
|
|
||||||
|
|
||||||
### Desktop
|
|
||||||
|
|
||||||
Desktop verwendet primär:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Sidebar
|
|
||||||
+ Hauptinhalt
|
|
||||||
+ optionaler Detailbereich
|
|
||||||
```
|
|
||||||
|
|
||||||
Beispiel:
|
|
||||||
|
|
||||||
```text
|
|
||||||
┌───────────────┬──────────────────────────────────────┐
|
|
||||||
│ Buying Agent │ Projekt: Mobiler Entwicklungsrechner│
|
|
||||||
│ │ │
|
|
||||||
│ Projekte │ Status / Suche / Ranking / Shortlist│
|
|
||||||
│ Admin │ │
|
|
||||||
│ Einstellungen │ │
|
|
||||||
└───────────────┴──────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mobile
|
|
||||||
|
|
||||||
Mobile verwendet primär:
|
|
||||||
|
|
||||||
- Bottom Navigation oder kompakte Hauptnavigation
|
|
||||||
- Vollbild-Detailseiten
|
|
||||||
- Tabs innerhalb eines BuyingProjects
|
|
||||||
- keine verkleinerte Desktop-Sidebar
|
|
||||||
|
|
||||||
Beispiel:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Projekt
|
|
||||||
────────────────
|
|
||||||
Übersicht
|
|
||||||
Kandidaten
|
|
||||||
Shortlist
|
|
||||||
Kriterien
|
|
||||||
Historie
|
|
||||||
|
|
||||||
[ Projekte ] [ Suche ] [ Admin ]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mitai-Bezug
|
|
||||||
|
|
||||||
Falls im bestehenden Mitai-Code wiederverwendbare Layout-, PWA-, Responsive- oder Navigationsmuster vorhanden sind, sollen diese bevorzugt übernommen oder konsistent nachgebildet werden.
|
|
||||||
|
|
||||||
Keine fachliche Kopplung an Mitai.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. Hauptnavigation
|
|
||||||
|
|
||||||
Der MVP benötigt mindestens:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Projekte
|
|
||||||
Admin
|
|
||||||
```
|
|
||||||
|
|
||||||
Optional:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Systemstatus
|
|
||||||
```
|
|
||||||
|
|
||||||
Noch nicht erforderlich:
|
|
||||||
|
|
||||||
- Benutzerverwaltung
|
|
||||||
- Warenkorb
|
|
||||||
- Benachrichtigungseinstellungen
|
|
||||||
- Account-/Profilverwaltung
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 6. Screen 1 – BuyingProject-Liste
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/projects
|
|
||||||
```
|
|
||||||
|
|
||||||
Funktionen:
|
|
||||||
|
|
||||||
- aktive BuyingProjects anzeigen
|
|
||||||
- Projektname
|
|
||||||
- Kategorie
|
|
||||||
- Status
|
|
||||||
- letzter erfolgreicher SearchRun
|
|
||||||
- Anzahl Kandidaten
|
|
||||||
- Anzahl Shortlist-Kandidaten
|
|
||||||
- aktueller Discovery-Status
|
|
||||||
- Button „Neues Projekt“
|
|
||||||
|
|
||||||
Mögliche Status:
|
|
||||||
|
|
||||||
```text
|
|
||||||
ACTIVE
|
|
||||||
PURCHASED
|
|
||||||
ARCHIVED
|
|
||||||
OUT_OF_SYNC
|
|
||||||
BLOCKED
|
|
||||||
TOO_BROAD
|
|
||||||
```
|
|
||||||
|
|
||||||
Die Liste darf keine produktkategoriespezifischen Felder voraussetzen.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 7. Screen 2 – BuyingProject anlegen
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/projects/new
|
|
||||||
```
|
|
||||||
|
|
||||||
### Schritt 1 – Basisdaten
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
```text
|
|
||||||
name
|
|
||||||
source
|
|
||||||
source_category
|
|
||||||
market
|
|
||||||
currency
|
|
||||||
shortlist_size
|
|
||||||
schedule_interval
|
|
||||||
```
|
|
||||||
|
|
||||||
Default:
|
|
||||||
|
|
||||||
```text
|
|
||||||
shortlist_size = 5
|
|
||||||
schedule_interval_minutes = 180
|
|
||||||
```
|
|
||||||
|
|
||||||
### Schritt 2 – Kriterien
|
|
||||||
|
|
||||||
Nach Auswahl der Source-Kategorie lädt das Frontend dynamisch:
|
|
||||||
|
|
||||||
- Canonical Criteria
|
|
||||||
- Source Filter Definitions
|
|
||||||
- CriterionMappings
|
|
||||||
- Mapping-Status
|
|
||||||
|
|
||||||
Das Frontend erzeugt daraus dynamisch die Eingabefelder.
|
|
||||||
|
|
||||||
Es gibt **keine fest codierten Notebook-Felder**.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 8. Dynamische Kriterien
|
|
||||||
|
|
||||||
Verfügbare Kriterien entstehen aus:
|
|
||||||
|
|
||||||
```text
|
|
||||||
SourceCategory
|
|
||||||
↓
|
|
||||||
SourceFilterDefinitions
|
|
||||||
↓
|
|
||||||
CriterionDefinitions
|
|
||||||
↓
|
|
||||||
CriterionMappings
|
|
||||||
```
|
|
||||||
|
|
||||||
Das Frontend rendert Controls anhand des Typs.
|
|
||||||
|
|
||||||
### Unterstützte Control-Typen
|
|
||||||
|
|
||||||
```text
|
|
||||||
BOOLEAN
|
|
||||||
ENUM
|
|
||||||
MULTI_ENUM
|
|
||||||
NUMERIC_RANGE
|
|
||||||
NUMERIC_THRESHOLD
|
|
||||||
TEXT
|
|
||||||
```
|
|
||||||
|
|
||||||
Für ein Kriterium sollen mindestens konfigurierbar sein:
|
|
||||||
|
|
||||||
```text
|
|
||||||
operator
|
|
||||||
value
|
|
||||||
mandatory / optional
|
|
||||||
```
|
|
||||||
|
|
||||||
Wenn vom Scoring unterstützt zusätzlich:
|
|
||||||
|
|
||||||
```text
|
|
||||||
weight
|
|
||||||
preference strategy
|
|
||||||
acceptable
|
|
||||||
ideal
|
|
||||||
direction
|
|
||||||
```
|
|
||||||
|
|
||||||
### Sichtbarer Mapping-Status
|
|
||||||
|
|
||||||
Pro Kriterium:
|
|
||||||
|
|
||||||
```text
|
|
||||||
VERIFIED
|
|
||||||
LEARNED_UNVERIFIED
|
|
||||||
INTERNAL_ONLY
|
|
||||||
BROKEN
|
|
||||||
DISABLED
|
|
||||||
```
|
|
||||||
|
|
||||||
Standardnutzer sollen primär verständliche Hinweise sehen:
|
|
||||||
|
|
||||||
- „Wird direkt bei Geizhals gefiltert“
|
|
||||||
- „Wird erst intern bewertet“
|
|
||||||
- „Filter derzeit nicht verfügbar“
|
|
||||||
|
|
||||||
Technische IDs bleiben im Admin-Bereich.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 9. Kriterienänderungen
|
|
||||||
|
|
||||||
Wenn Kriterien geändert werden:
|
|
||||||
|
|
||||||
```text
|
|
||||||
criteria_version++
|
|
||||||
```
|
|
||||||
|
|
||||||
Die bestehende DiscoverySearch wird:
|
|
||||||
|
|
||||||
```text
|
|
||||||
OUT_OF_SYNC
|
|
||||||
```
|
|
||||||
|
|
||||||
Danach muss das Backend eine neue Query aus den aktuellen:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Criteria
|
|
||||||
+
|
|
||||||
VERIFIED Mappings
|
|
||||||
```
|
|
||||||
|
|
||||||
erzeugen.
|
|
||||||
|
|
||||||
Das Frontend zeigt klar:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Suchkriterien wurden geändert.
|
|
||||||
Die externe Suche muss aktualisiert werden.
|
|
||||||
```
|
|
||||||
|
|
||||||
Aktion:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Suche aktualisieren
|
|
||||||
```
|
|
||||||
|
|
||||||
Danach:
|
|
||||||
|
|
||||||
```text
|
|
||||||
generate-query
|
|
||||||
→ optional run
|
|
||||||
→ neuer SearchRun
|
|
||||||
→ neues Ranking
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 10. Screen 3 – BuyingProject-Detail
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/projects/:projectId
|
|
||||||
```
|
|
||||||
|
|
||||||
Desktop: strukturierte Detailansicht.
|
|
||||||
|
|
||||||
Mobile: Tabs.
|
|
||||||
|
|
||||||
Mindestens folgende Bereiche:
|
|
||||||
|
|
||||||
### Übersicht
|
|
||||||
|
|
||||||
- Projektstatus
|
|
||||||
- Kategorie
|
|
||||||
- letzter SearchRun
|
|
||||||
- Discovery-Status
|
|
||||||
- Trefferzahl
|
|
||||||
- Kandidatenzahl
|
|
||||||
- Shortlist-Größe
|
|
||||||
- Zeitpunkt nächste geplante Suche
|
|
||||||
- wichtige Warnungen
|
|
||||||
|
|
||||||
### Kandidaten
|
|
||||||
|
|
||||||
- aktuelle Kandidaten
|
|
||||||
- ProductScore
|
|
||||||
- product_eligibility
|
|
||||||
- aktueller best_price, sofern vorhanden
|
|
||||||
- Diff-Status
|
|
||||||
- Lifecycle-Status
|
|
||||||
- Shortlist-Markierung
|
|
||||||
|
|
||||||
### Shortlist
|
|
||||||
|
|
||||||
Top-N gemäß Projektkonfiguration.
|
|
||||||
|
|
||||||
### Kriterien
|
|
||||||
|
|
||||||
Dynamische Kriterien anzeigen und bearbeiten.
|
|
||||||
|
|
||||||
### Historie
|
|
||||||
|
|
||||||
- SearchRuns
|
|
||||||
- wichtige Statusänderungen
|
|
||||||
- bisherige Rankings
|
|
||||||
- soweit vorhanden frühere Bewertungen / Empfehlungen
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 11. Screen 4 – Kandidatenliste
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/projects/:projectId/candidates
|
|
||||||
```
|
|
||||||
|
|
||||||
Darstellung:
|
|
||||||
|
|
||||||
- sortierbare Liste/Tabelle auf Desktop
|
|
||||||
- Cards/List Items auf Mobile
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Produktname
|
|
||||||
Hersteller
|
|
||||||
wichtige dynamische Attribute
|
|
||||||
best_price
|
|
||||||
ProductScore
|
|
||||||
product_eligibility
|
|
||||||
diff_status
|
|
||||||
lifecycle_status
|
|
||||||
shortlist_status
|
|
||||||
last_seen
|
|
||||||
```
|
|
||||||
|
|
||||||
### Dynamische Attribute
|
|
||||||
|
|
||||||
Welche Eigenschaften sichtbar sind, darf nicht notebook-spezifisch hart codiert sein.
|
|
||||||
|
|
||||||
Mögliche Strategie:
|
|
||||||
|
|
||||||
1. Source-/Category-Definition liefert relevante display attributes.
|
|
||||||
2. Projekt kann später bevorzugte sichtbare Attribute konfigurieren.
|
|
||||||
3. MVP darf zunächst die wichtigsten vorhandenen normalisierten Attribute anzeigen.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 12. Screen 5 – Kandidatendetail
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/projects/:projectId/candidates/:candidateId
|
|
||||||
```
|
|
||||||
|
|
||||||
Der Nutzer soll in ein Produkt „hineinspringen“ können.
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
### Aktueller Status
|
|
||||||
|
|
||||||
- Name
|
|
||||||
- Hersteller
|
|
||||||
- Source-Link
|
|
||||||
- ProductScore
|
|
||||||
- product_eligibility
|
|
||||||
- Shortlist-Status
|
|
||||||
- aktueller best_price
|
|
||||||
- letzte Sichtung
|
|
||||||
|
|
||||||
### Attribute
|
|
||||||
|
|
||||||
Alle verfügbaren normalisierten dynamischen Produktattribute.
|
|
||||||
|
|
||||||
Keine notebook-spezifische Feldliste.
|
|
||||||
|
|
||||||
### Historie
|
|
||||||
|
|
||||||
- Preisentwicklung aus ProductObservations
|
|
||||||
- relevante Attributänderungen
|
|
||||||
- NEW / CHANGED / MISSING / DISAPPEARED
|
|
||||||
- SearchRun-Bezug
|
|
||||||
|
|
||||||
### Bewertungen / Ergebnisse
|
|
||||||
|
|
||||||
- aktuelle Bewertung
|
|
||||||
- frühere gespeicherte Bewertungen, soweit Backend vorhanden
|
|
||||||
- Gründe / Constraint-Ergebnisse
|
|
||||||
- Coverage / fehlende Daten
|
|
||||||
|
|
||||||
### Spätere Erweiterbarkeit
|
|
||||||
|
|
||||||
Dieser Screen wird später Zielort für:
|
|
||||||
|
|
||||||
- Deep Enrichment
|
|
||||||
- Tests
|
|
||||||
- Reviews
|
|
||||||
- Benchmarks
|
|
||||||
- Händlervergleich
|
|
||||||
- Kaufempfehlung
|
|
||||||
|
|
||||||
Diese Funktionen sind noch nicht Bestandteil dieses MVP-Slices.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 13. Screen 6 – Shortlist
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/projects/:projectId/shortlist
|
|
||||||
```
|
|
||||||
|
|
||||||
Zweck:
|
|
||||||
|
|
||||||
Nur die aktuell relevantesten Produkte zeigen.
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
- Rang
|
|
||||||
- Produkt
|
|
||||||
- ProductScore
|
|
||||||
- Product Eligibility
|
|
||||||
- aktueller Preis
|
|
||||||
- relevante Stärken
|
|
||||||
- relevante Schwächen / fehlende Daten
|
|
||||||
- letzter Änderungsstatus
|
|
||||||
|
|
||||||
Default:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Top 5
|
|
||||||
```
|
|
||||||
|
|
||||||
konfigurierbar am BuyingProject.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 14. Screen 7 – Search Runs / Historie
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/projects/:projectId/history
|
|
||||||
```
|
|
||||||
|
|
||||||
Anzeige:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Zeitpunkt
|
|
||||||
Status
|
|
||||||
Trefferzahl
|
|
||||||
Page Count
|
|
||||||
NEW
|
|
||||||
CHANGED
|
|
||||||
MISSING
|
|
||||||
DISAPPEARED
|
|
||||||
TOO_BROAD
|
|
||||||
Fehler / BLOCKED
|
|
||||||
```
|
|
||||||
|
|
||||||
Ein SearchRun muss geöffnet werden können, um Detailinformationen zu sehen.
|
|
||||||
|
|
||||||
Kein HTML anzeigen oder archivieren.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 15. Screen 8 – Admin: Taxonomie & Mapping
|
|
||||||
|
|
||||||
Route:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/admin/sources
|
|
||||||
/admin/sources/:source/categories/:category
|
|
||||||
```
|
|
||||||
|
|
||||||
Dieser Bereich darf technischer sein.
|
|
||||||
|
|
||||||
Funktionen:
|
|
||||||
|
|
||||||
- gelernte Kategorien sehen
|
|
||||||
- SourceFilterDefinitions sehen
|
|
||||||
- SourceFilterOptions/Ranges sehen
|
|
||||||
- Canonical Criteria sehen
|
|
||||||
- CriterionMappings sehen
|
|
||||||
- Mapping-Status ändern
|
|
||||||
- fehlerhafte kleine Mapping-Änderungen direkt korrigieren
|
|
||||||
- BROKEN / DISABLED erkennen
|
|
||||||
- Taxonomie neu einlesen
|
|
||||||
- Query-Mapping nachvollziehen
|
|
||||||
|
|
||||||
### Wichtig
|
|
||||||
|
|
||||||
Das Frontend darf kein URL-Encoding selbst erfinden.
|
|
||||||
|
|
||||||
Es editiert/pflegt nur persistierte Mapping-Daten.
|
|
||||||
|
|
||||||
Backend bleibt für Validierung und Query-Generation verantwortlich.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 16. Server State und Data Fetching
|
|
||||||
|
|
||||||
Empfehlung:
|
|
||||||
|
|
||||||
```text
|
|
||||||
TanStack Query
|
|
||||||
```
|
|
||||||
|
|
||||||
für:
|
|
||||||
|
|
||||||
- Projekte
|
|
||||||
- Kriterien
|
|
||||||
- Taxonomie
|
|
||||||
- Mappings
|
|
||||||
- SearchRuns
|
|
||||||
- Kandidaten
|
|
||||||
- Shortlist
|
|
||||||
- Produktdetails
|
|
||||||
|
|
||||||
Client-State nur für:
|
|
||||||
|
|
||||||
- UI-Zustände
|
|
||||||
- Formulare
|
|
||||||
- Tabs
|
|
||||||
- lokale Filter
|
|
||||||
|
|
||||||
Keine fachlich relevante Server-Wahrheit ausschließlich im Browser halten.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 17. API-Verträge
|
|
||||||
|
|
||||||
Das Frontend nutzt mindestens:
|
|
||||||
|
|
||||||
### Projekte
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/buying-projects
|
|
||||||
POST /api/v1/buying-projects
|
|
||||||
GET /api/v1/buying-projects/{id}
|
|
||||||
PATCH /api/v1/buying-projects/{id}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Projektkriterien
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/buying-projects/{id}/criteria
|
|
||||||
PUT /api/v1/buying-projects/{id}/criteria
|
|
||||||
```
|
|
||||||
|
|
||||||
Falls Backend intern versionierte Ressourcen bevorzugt, kann PUT eine neue Version erzeugen.
|
|
||||||
|
|
||||||
### Taxonomie
|
|
||||||
|
|
||||||
```text
|
|
||||||
POST /api/v1/sources/{source}/categories/{category}/discover-taxonomy
|
|
||||||
GET /api/v1/sources/{source}/categories/{category}/filters
|
|
||||||
```
|
|
||||||
|
|
||||||
### Criterion Definitions / Mappings
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/criterion-definitions
|
|
||||||
GET /api/v1/criterion-mappings
|
|
||||||
POST /api/v1/criterion-mappings
|
|
||||||
PATCH /api/v1/criterion-mappings/{id}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Discovery Search
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/buying-projects/{id}/discovery-search
|
|
||||||
POST /api/v1/discovery-searches/{id}/generate-query
|
|
||||||
POST /api/v1/discovery-searches/{id}/run
|
|
||||||
```
|
|
||||||
|
|
||||||
### Search Runs
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/discovery-searches/{id}/runs
|
|
||||||
GET /api/v1/search-runs/{id}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Candidates
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/buying-projects/{id}/candidates
|
|
||||||
GET /api/v1/buying-projects/{id}/shortlist
|
|
||||||
GET /api/v1/buying-projects/{id}/candidates/{candidateId}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 18. Status- und Fehlerdarstellung
|
|
||||||
|
|
||||||
Frontend muss mindestens folgende Zustände sichtbar machen:
|
|
||||||
|
|
||||||
### Discovery
|
|
||||||
|
|
||||||
```text
|
|
||||||
SUCCESS
|
|
||||||
PARTIAL
|
|
||||||
FAILED
|
|
||||||
BLOCKED
|
|
||||||
TOO_BROAD
|
|
||||||
OUT_OF_SYNC
|
|
||||||
```
|
|
||||||
|
|
||||||
### Candidate Diff
|
|
||||||
|
|
||||||
```text
|
|
||||||
NEW
|
|
||||||
CHANGED
|
|
||||||
UNCHANGED
|
|
||||||
MISSING
|
|
||||||
DISAPPEARED
|
|
||||||
```
|
|
||||||
|
|
||||||
### Matching
|
|
||||||
|
|
||||||
```text
|
|
||||||
MATCHED
|
|
||||||
MATCH_UNCERTAIN
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mapping
|
|
||||||
|
|
||||||
```text
|
|
||||||
DISCOVERED
|
|
||||||
LEARNED_UNVERIFIED
|
|
||||||
VERIFIED
|
|
||||||
BROKEN
|
|
||||||
DISABLED
|
|
||||||
INTERNAL_ONLY
|
|
||||||
```
|
|
||||||
|
|
||||||
Fehlertexte für Standardnutzer verständlich formulieren.
|
|
||||||
|
|
||||||
Technische Details optional aufklappbar.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 19. Loading / Empty / Error States
|
|
||||||
|
|
||||||
Jeder Screen braucht explizit:
|
|
||||||
|
|
||||||
- Loading
|
|
||||||
- Empty
|
|
||||||
- Error
|
|
||||||
- Retry
|
|
||||||
|
|
||||||
Beispiele:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Noch keine Kandidaten gefunden.
|
|
||||||
Noch kein SearchRun ausgeführt.
|
|
||||||
Taxonomie wurde noch nicht geladen.
|
|
||||||
Diese Suche ist derzeit BLOCKED.
|
|
||||||
Die Kriterien wurden geändert; Suche ist OUT_OF_SYNC.
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 20. PWA-Grundgerüst
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
- installierbare PWA
|
|
||||||
- Manifest
|
|
||||||
- Icons / App-Metadaten
|
|
||||||
- responsive shell
|
|
||||||
- app-like navigation
|
|
||||||
- sinnvolle Offline-Fallback-Seite
|
|
||||||
|
|
||||||
Keine Offline-Businesslogik erforderlich.
|
|
||||||
|
|
||||||
Keine komplexe Synchronisationslogik in v0.1.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 21. Tests
|
|
||||||
|
|
||||||
### Unit / Component
|
|
||||||
|
|
||||||
- dynamischer Criterion Renderer je Filtertyp
|
|
||||||
- Mapping-Status-Darstellung
|
|
||||||
- Candidate Status-Darstellung
|
|
||||||
- responsive Komponenten
|
|
||||||
- Formvalidierung
|
|
||||||
|
|
||||||
### API Integration
|
|
||||||
|
|
||||||
- Projektliste
|
|
||||||
- Projektanlage
|
|
||||||
- Kriterien laden/speichern
|
|
||||||
- SearchRun starten
|
|
||||||
- Kandidaten laden
|
|
||||||
- Shortlist laden
|
|
||||||
- Admin-Mapping aktualisieren
|
|
||||||
|
|
||||||
### E2E
|
|
||||||
|
|
||||||
Mindestens ein vollständiger Notebook-Pilot:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Projekt anlegen
|
|
||||||
→ Kategorie Notebooks
|
|
||||||
→ dynamische Kriterien laden
|
|
||||||
→ Kriterien setzen
|
|
||||||
→ Geizhals Query generieren
|
|
||||||
→ SearchRun starten
|
|
||||||
→ Kandidaten erscheinen
|
|
||||||
→ Ranking sichtbar
|
|
||||||
→ Shortlist sichtbar
|
|
||||||
→ Kandidat öffnen
|
|
||||||
→ Historie sichtbar
|
|
||||||
→ Kriterium ändern
|
|
||||||
→ OUT_OF_SYNC sichtbar
|
|
||||||
→ Query neu erzeugen
|
|
||||||
```
|
|
||||||
|
|
||||||
### Responsive E2E
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
- Desktop-Viewport
|
|
||||||
- Mobile-Viewport
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 22. Nicht implementieren
|
|
||||||
|
|
||||||
In diesem Slice weiterhin nicht:
|
|
||||||
|
|
||||||
- Login/Auth
|
|
||||||
- RBAC
|
|
||||||
- Einladungen
|
|
||||||
- Multi-User-UI
|
|
||||||
- Keepa
|
|
||||||
- Idealo
|
|
||||||
- WhatsApp
|
|
||||||
- Warenkorb
|
|
||||||
- Kaufautomatisierung
|
|
||||||
- LLM-Automation
|
|
||||||
- Deep Enrichment
|
|
||||||
- finale Decision Engine
|
|
||||||
|
|
||||||
Die Architektur darf diese Erweiterungen nicht verbauen.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 23. Definition of Done
|
|
||||||
|
|
||||||
- [ ] React + TypeScript + Vite Frontend existiert im Monorepo.
|
|
||||||
- [ ] PWA ist installierbar.
|
|
||||||
- [ ] Desktop- und Mobile-Layouts sind bewusst getrennt gestaltet.
|
|
||||||
- [ ] BuyingProjects können über die UI angelegt und geöffnet werden.
|
|
||||||
- [ ] Produktkategorie kann ausgewählt werden.
|
|
||||||
- [ ] Kriterien werden dynamisch aus Taxonomie / CriterionDefinitions erzeugt.
|
|
||||||
- [ ] Keine notebook-spezifischen Criteria-Felder im Frontend-Core.
|
|
||||||
- [ ] Kriterien können bearbeitet und gespeichert werden.
|
|
||||||
- [ ] Mapping-Status ist sichtbar.
|
|
||||||
- [ ] INTERNAL_ONLY-Kriterien können gespeichert werden.
|
|
||||||
- [ ] Discovery Search kann generiert werden.
|
|
||||||
- [ ] SearchRun kann aus der UI gestartet werden.
|
|
||||||
- [ ] SearchRun-Status ist sichtbar.
|
|
||||||
- [ ] Kandidatenliste funktioniert.
|
|
||||||
- [ ] Ranking ist sichtbar.
|
|
||||||
- [ ] Shortlist funktioniert.
|
|
||||||
- [ ] Kandidatendetail zeigt dynamische Attribute und Historie.
|
|
||||||
- [ ] Projekt-Historie zeigt SearchRuns und relevante Änderungen.
|
|
||||||
- [ ] Kriterienänderung setzt/zeigt OUT_OF_SYNC und ermöglicht Regeneration.
|
|
||||||
- [ ] Admin-Bereich erlaubt Prüfung/Korrektur von Source-Taxonomie und Mappings.
|
|
||||||
- [ ] Loading-/Empty-/Error-Zustände existieren.
|
|
||||||
- [ ] E2E-Test des vollständigen Notebook-Piloten ist grün.
|
|
||||||
- [ ] Desktop- und Mobile-E2E sind grün.
|
|
||||||
- [ ] Backend bleibt alleinige Quelle fachlicher Wahrheit.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# Coding-Agent-Arbeitsauftrag
|
|
||||||
|
|
||||||
Verwende als verbindliche Anforderungen:
|
|
||||||
|
|
||||||
1. `buying-agent-v0.1-spec-v1.1.md`
|
|
||||||
2. `buying-agent-architecture-addendum-v1.md`
|
|
||||||
3. `buying-agent-source-discovery-spec-v0.1.md`
|
|
||||||
4. `buying-agent-frontend-mvp-spec-v0.1.md`
|
|
||||||
|
|
||||||
Die Frontend-MVP-Spec hebt für den nutzbaren MVP die frühere Scope-Aussage „kein Frontend“ auf.
|
|
||||||
|
|
||||||
## 1. Vor Implementierung
|
|
||||||
|
|
||||||
Prüfe den aktuellen Repository-Stand gegen alle vier Dokumente.
|
|
||||||
|
|
||||||
Berichte kurz:
|
|
||||||
|
|
||||||
- welche Backend-APIs für den Frontend-Flow bereits existieren,
|
|
||||||
- welche Endpunkte fehlen,
|
|
||||||
- ob aktuelle DTOs dynamische Kriterien und Produktattribute vollständig tragen,
|
|
||||||
- ob bestehender Code notebook-spezifische Felder enthält,
|
|
||||||
- welche Änderungen rein additiv möglich sind.
|
|
||||||
|
|
||||||
Keine größere strukturelle Abweichung ohne Rückfrage.
|
|
||||||
|
|
||||||
## 2. Vertikalen MVP implementieren
|
|
||||||
|
|
||||||
Nicht Backend und Frontend getrennt in langen Phasen bauen.
|
|
||||||
|
|
||||||
Stattdessen vertikal:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Projektliste
|
|
||||||
→ Projektanlage
|
|
||||||
→ dynamische Kriterien
|
|
||||||
→ Query Generation
|
|
||||||
→ SearchRun
|
|
||||||
→ Kandidaten
|
|
||||||
→ Ranking
|
|
||||||
→ Shortlist
|
|
||||||
→ Kandidatendetail
|
|
||||||
```
|
|
||||||
|
|
||||||
Jeder Schritt soll nach Implementierung unmittelbar über die UI prüfbar sein.
|
|
||||||
|
|
||||||
## 3. Dynamik sicherstellen
|
|
||||||
|
|
||||||
Keine festen Felder wie:
|
|
||||||
|
|
||||||
```text
|
|
||||||
ram_gb
|
|
||||||
storage_gb
|
|
||||||
weight_kg
|
|
||||||
display_inches
|
|
||||||
```
|
|
||||||
|
|
||||||
in generischen Frontend-Komponenten hardcoden.
|
|
||||||
|
|
||||||
Controls und sichtbare Produktattribute werden metadata-driven erzeugt.
|
|
||||||
|
|
||||||
Notebook-spezifische Werte dürfen nur als gelernte/gespeicherte Daten vorkommen.
|
|
||||||
|
|
||||||
## 4. UI
|
|
||||||
|
|
||||||
- React + TypeScript + Vite
|
|
||||||
- PWA
|
|
||||||
- TanStack Query
|
|
||||||
- einfache, nüchterne Bedienung
|
|
||||||
- Desktop: Sidebar + Content/Detail
|
|
||||||
- Mobile: kompakte Navigation / Bottom Navigation + Vollbild-Details
|
|
||||||
- vorhandene Mitai-Muster wiederverwenden, falls im zugänglichen Repo vorhanden und passend
|
|
||||||
|
|
||||||
## 5. Tests
|
|
||||||
|
|
||||||
Führe aus:
|
|
||||||
|
|
||||||
- Frontend Unit/Component Tests
|
|
||||||
- API Integration Tests
|
|
||||||
- E2E vollständiger Notebook-Pilot
|
|
||||||
- Desktop E2E
|
|
||||||
- Mobile E2E
|
|
||||||
|
|
||||||
## 6. Abschlussbericht
|
|
||||||
|
|
||||||
Dokumentiere:
|
|
||||||
|
|
||||||
- implementierte Screens
|
|
||||||
- neue/angepasste Backend-Endpunkte
|
|
||||||
- verwendete Frontend-Bibliotheken
|
|
||||||
- responsive Verhalten
|
|
||||||
- ausgeführte Tests und Ergebnisse
|
|
||||||
- offene Abweichungen
|
|
||||||
- Risiken für spätere Multi-User-, Deep-Enrichment- und Decision-Erweiterungen
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,986 +0,0 @@
|
||||||
# Buying Agent – Source & Discovery Specification v0.1
|
|
||||||
|
|
||||||
**Status:** implementierungsbereit
|
|
||||||
**Spezifikationsversion:** 0.1
|
|
||||||
**Stand:** 2026-10-04
|
|
||||||
**Ergänzt:** `buying-agent-v0.1-spec-v1.1.md` und `buying-agent-architecture-addendum-v1.md`
|
|
||||||
|
|
||||||
## 1. Zweck
|
|
||||||
|
|
||||||
Diese Spezifikation definiert den ersten real nutzbaren Discovery-Slice des Buying Agents.
|
|
||||||
|
|
||||||
Ziel ist, externe Produktsuchen automatisiert aus den Kriterien eines BuyingProjects abzuleiten, regelmäßig auszuführen, Ergebnisse dauerhaft zu verwalten, Veränderungen zu erkennen und die Treffer anschließend durch die eigene Buying-Engine zu bewerten.
|
|
||||||
|
|
||||||
Der erste Pilot erfolgt mit **Geizhals, Kategorie Notebooks**. Die Architektur darf jedoch nicht notebook-spezifisch sein.
|
|
||||||
|
|
||||||
```text
|
|
||||||
BuyingProject
|
|
||||||
↓
|
|
||||||
Canonical Criteria
|
|
||||||
↓
|
|
||||||
Source Taxonomy + Mapping
|
|
||||||
↓
|
|
||||||
generierte Geizhals-Suche
|
|
||||||
↓
|
|
||||||
periodischer SearchRun
|
|
||||||
↓
|
|
||||||
persistierte Produkte + Beobachtungen
|
|
||||||
↓
|
|
||||||
Diff zum vorherigen Lauf
|
|
||||||
↓
|
|
||||||
NEW / CHANGED / MISSING / DISAPPEARED
|
|
||||||
↓
|
|
||||||
interne Constraints + Ranking
|
|
||||||
↓
|
|
||||||
Shortlist
|
|
||||||
↓
|
|
||||||
später Deep Enrichment
|
|
||||||
↓
|
|
||||||
später Decision
|
|
||||||
```
|
|
||||||
|
|
||||||
## 2. Fachliche Leitprinzipien
|
|
||||||
|
|
||||||
1. Geizhals ist Discovery-Quelle, nicht Decision Engine.
|
|
||||||
2. Die eigenen BuyingProject-Kriterien bleiben die fachliche Quelle der Wahrheit.
|
|
||||||
3. Geizhals wird als möglichst starker Vorfilter genutzt.
|
|
||||||
4. Nicht auf Geizhals abbildbare Kriterien bleiben interne Kriterien.
|
|
||||||
5. Discovery, Ranking und Decision bleiben getrennte fachliche Stufen.
|
|
||||||
6. Externe Filtertaxonomien werden **on demand** gelernt, nicht vollständig vorab katalogisiert.
|
|
||||||
7. URL-Encoding und Filtersyntax dürfen nicht vom LLM erfunden werden.
|
|
||||||
8. Produktidentität wird zunächst konservativ und pragmatisch behandelt.
|
|
||||||
9. Bei Unsicherheit erfolgt keine automatische Zusammenführung.
|
|
||||||
10. Suchläufe speichern strukturierte Zustände, keine HTML-Archive.
|
|
||||||
|
|
||||||
## 3. Scope v0.1
|
|
||||||
|
|
||||||
### Enthalten
|
|
||||||
|
|
||||||
- Geizhals als erste Discovery-Quelle
|
|
||||||
- Pilotkategorie Notebooks
|
|
||||||
- On-Demand-Erfassung der Geizhals-Filtertaxonomie
|
|
||||||
- Speicherung von Source-Kategorien, Filterdefinitionen, Optionen und Wertebereichen
|
|
||||||
- Mapping zwischen Canonical Criteria und Source-Filtern
|
|
||||||
- reproduzierbare Generierung von Geizhals-Such-URLs
|
|
||||||
- periodische SearchRuns
|
|
||||||
- vollständige Ergebnisaufnahme einer aktiven Suche
|
|
||||||
- persistierte SourceProductReferences
|
|
||||||
- Historisierung sichtbarer Attribute und Preise
|
|
||||||
- Diffing zwischen Suchläufen
|
|
||||||
- Status `NEW`, `CHANGED`, `UNCHANGED`, `MISSING`, `DISAPPEARED`
|
|
||||||
- interne Bewertung der Kandidaten
|
|
||||||
- konfigurierbare Shortlist, Default 5
|
|
||||||
- defensives Fehler-/Backoff-Verhalten
|
|
||||||
- Tests
|
|
||||||
|
|
||||||
### Nicht enthalten
|
|
||||||
|
|
||||||
- Frontend-Implementierung
|
|
||||||
- Keepa
|
|
||||||
- Idealo
|
|
||||||
- LLM-Automation
|
|
||||||
- automatische Kaufentscheidung
|
|
||||||
- Kaufautomatisierung
|
|
||||||
- Warenkorbmanagement
|
|
||||||
- Authentifizierung / RBAC
|
|
||||||
- Multi-User-UI
|
|
||||||
- automatische Deep-Enrichment-Quellen
|
|
||||||
- Umgehung von Schutzmechanismen externer Websites
|
|
||||||
|
|
||||||
## 4. Source Taxonomy
|
|
||||||
|
|
||||||
### 4.1 SourceCategory
|
|
||||||
|
|
||||||
Repräsentiert eine Kategorie einer externen Quelle.
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"source": "geizhals",
|
|
||||||
"external_category_key": "nb",
|
|
||||||
"name": "Notebooks",
|
|
||||||
"source_url": "https://geizhals.de/?cat=nb",
|
|
||||||
"status": "ACTIVE",
|
|
||||||
"last_verified_at": "2026-10-04T10:00:00Z"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4.2 SourceFilterDefinition
|
|
||||||
|
|
||||||
Beschreibt einen von Geizhals angebotenen Filter.
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"source": "geizhals",
|
|
||||||
"category_key": "nb",
|
|
||||||
"external_filter_key": "17023",
|
|
||||||
"label": "Arbeitsspeicher",
|
|
||||||
"filter_type": "NUMERIC_THRESHOLD",
|
|
||||||
"canonical_unit": "GB",
|
|
||||||
"status": "VERIFIED"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Mögliche `filter_type`-Werte:
|
|
||||||
|
|
||||||
```text
|
|
||||||
BOOLEAN
|
|
||||||
ENUM
|
|
||||||
MULTI_ENUM
|
|
||||||
NUMERIC_RANGE
|
|
||||||
NUMERIC_THRESHOLD
|
|
||||||
TEXT
|
|
||||||
UNKNOWN
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4.3 SourceFilterOption
|
|
||||||
|
|
||||||
Für diskrete Filterwerte:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"filter_key": "17023",
|
|
||||||
"source_value": "ab 24GB",
|
|
||||||
"normalized_value": 24,
|
|
||||||
"operator": ">=",
|
|
||||||
"unit": "GB"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4.4 SourceFilterRange
|
|
||||||
|
|
||||||
Für Bereiche bzw. frei parametrisierbare Werte:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"filter_key": "price",
|
|
||||||
"minimum": 0,
|
|
||||||
"maximum": null,
|
|
||||||
"step": 1,
|
|
||||||
"unit": "EUR"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 5. On-Demand Taxonomy Learning
|
|
||||||
|
|
||||||
Die Taxonomie wird nur für tatsächlich verwendete Kategorien gelernt.
|
|
||||||
|
|
||||||
Initial:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Geizhals → Notebooks
|
|
||||||
```
|
|
||||||
|
|
||||||
Später z. B.:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Monitore
|
|
||||||
Fernseher
|
|
||||||
Kameras
|
|
||||||
Rasenmäher
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
### Ablauf
|
|
||||||
|
|
||||||
```text
|
|
||||||
Kategorie öffnen
|
|
||||||
↓
|
|
||||||
verfügbare Filter erkennen
|
|
||||||
↓
|
|
||||||
Filterdefinitionen speichern
|
|
||||||
↓
|
|
||||||
Optionen / Bereiche erfassen
|
|
||||||
↓
|
|
||||||
URL-Verhalten beobachten
|
|
||||||
↓
|
|
||||||
Mapping erzeugen
|
|
||||||
↓
|
|
||||||
technisch verifizieren
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mapping-Status
|
|
||||||
|
|
||||||
```text
|
|
||||||
DISCOVERED
|
|
||||||
LEARNED_UNVERIFIED
|
|
||||||
VERIFIED
|
|
||||||
BROKEN
|
|
||||||
DISABLED
|
|
||||||
```
|
|
||||||
|
|
||||||
Nur `VERIFIED` darf für automatische produktive Suchläufe verwendet werden.
|
|
||||||
|
|
||||||
### Änderbarkeit
|
|
||||||
|
|
||||||
Die gelernten Mappings sollen später im Frontend administrierbar sein.
|
|
||||||
|
|
||||||
v0.1 benötigt dafür noch keine UI, aber die Persistenz muss Änderungen ermöglichen, ohne Source-Code anzupassen.
|
|
||||||
|
|
||||||
## 6. Canonical Criteria
|
|
||||||
|
|
||||||
Canonical Criteria sind die interne fachliche Darstellung eines Suchkriteriums.
|
|
||||||
|
|
||||||
Sie sollen sich stark an der Source-Taxonomie orientieren, aber nicht fest an Geizhals gekoppelt sein.
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"key": "ram_gb",
|
|
||||||
"value_type": "number",
|
|
||||||
"unit": "GB",
|
|
||||||
"semantic": "working_memory"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Ein BuyingProject verwendet daraus konkrete Anforderungen:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"criterion": "ram_gb",
|
|
||||||
"operator": ">=",
|
|
||||||
"value": 24,
|
|
||||||
"mandatory": true
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Source Mapping
|
|
||||||
|
|
||||||
```text
|
|
||||||
Canonical Criterion
|
|
||||||
ram_gb >= 24
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
Geizhals Mapping
|
|
||||||
filter_key = 17023
|
|
||||||
source_value = "ab 24GB"
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
URL fragment
|
|
||||||
xf=17023_ab+24GB
|
|
||||||
```
|
|
||||||
|
|
||||||
Nicht jedes Canonical Criterion muss ein Source Mapping besitzen.
|
|
||||||
|
|
||||||
Fehlt ein Mapping:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Criterion → INTERNAL_ONLY
|
|
||||||
```
|
|
||||||
|
|
||||||
Das Kriterium wird dann erst nach Discovery intern bewertet.
|
|
||||||
|
|
||||||
## 7. DiscoverySearch
|
|
||||||
|
|
||||||
Eine `DiscoverySearch` gehört zu einem BuyingProject und einer Source.
|
|
||||||
|
|
||||||
Sie enthält mindestens:
|
|
||||||
|
|
||||||
```text
|
|
||||||
id
|
|
||||||
buying_project_id
|
|
||||||
source_id
|
|
||||||
source_category_id
|
|
||||||
criteria_version
|
|
||||||
mapping_version
|
|
||||||
generated_url
|
|
||||||
status
|
|
||||||
schedule_interval_minutes
|
|
||||||
broad_result_threshold
|
|
||||||
created_at
|
|
||||||
updated_at
|
|
||||||
last_successful_run_at
|
|
||||||
```
|
|
||||||
|
|
||||||
Defaults:
|
|
||||||
|
|
||||||
```text
|
|
||||||
schedule_interval_minutes = 180
|
|
||||||
shortlist_size = 5
|
|
||||||
broad_result_threshold = 300
|
|
||||||
```
|
|
||||||
|
|
||||||
## 8. Search URL Generation
|
|
||||||
|
|
||||||
Die URL wird ausschließlich aus:
|
|
||||||
|
|
||||||
```text
|
|
||||||
BuyingProject Criteria
|
|
||||||
+
|
|
||||||
VERIFIED CriterionMappings
|
|
||||||
+
|
|
||||||
SourceCategory
|
|
||||||
+
|
|
||||||
konfigurierbaren Source-Optionen
|
|
||||||
```
|
|
||||||
|
|
||||||
generiert.
|
|
||||||
|
|
||||||
Die URL ist abgeleitet, nicht die fachliche Quelle der Wahrheit.
|
|
||||||
|
|
||||||
Eine Änderung der BuyingProject-Kriterien erzeugt eine neue `criteria_version`.
|
|
||||||
|
|
||||||
Wenn die gespeicherte DiscoverySearch nicht mehr zur aktuellen Criteria-/Mapping-Version passt:
|
|
||||||
|
|
||||||
```text
|
|
||||||
status = OUT_OF_SYNC
|
|
||||||
```
|
|
||||||
|
|
||||||
Danach muss die URL neu generiert werden.
|
|
||||||
|
|
||||||
## 9. Ergebnisumfang
|
|
||||||
|
|
||||||
Aktive Suchläufe werden grundsätzlich vollständig eingelesen.
|
|
||||||
|
|
||||||
Keine stille Begrenzung auf Top-N.
|
|
||||||
|
|
||||||
Nur vollständige Ergebnismengen erlauben belastbare Erkennung von:
|
|
||||||
|
|
||||||
```text
|
|
||||||
NEW
|
|
||||||
CHANGED
|
|
||||||
MISSING
|
|
||||||
DISAPPEARED
|
|
||||||
```
|
|
||||||
|
|
||||||
### TOO_BROAD
|
|
||||||
|
|
||||||
Überschreitet die Treffermenge den konfigurierten Schwellwert:
|
|
||||||
|
|
||||||
```text
|
|
||||||
result_count > broad_result_threshold
|
|
||||||
```
|
|
||||||
|
|
||||||
wird die Suche markiert:
|
|
||||||
|
|
||||||
```text
|
|
||||||
TOO_BROAD
|
|
||||||
```
|
|
||||||
|
|
||||||
Die Suche darf technisch weiter ausgeführt werden, erzeugt aber einen Hinweis, dass die Source-Kriterien geschärft werden sollten.
|
|
||||||
|
|
||||||
Spätere Versionen dürfen breite Suchen differenziert behandeln, etwa seltenere Full-Reconciliation.
|
|
||||||
|
|
||||||
## 10. SearchRun
|
|
||||||
|
|
||||||
Jeder Abruf erzeugt einen unveränderlichen `SearchRun`.
|
|
||||||
|
|
||||||
Felder:
|
|
||||||
|
|
||||||
```text
|
|
||||||
id
|
|
||||||
discovery_search_id
|
|
||||||
criteria_version
|
|
||||||
mapping_version
|
|
||||||
generated_url
|
|
||||||
started_at
|
|
||||||
finished_at
|
|
||||||
status
|
|
||||||
result_count
|
|
||||||
page_count
|
|
||||||
http_status_summary
|
|
||||||
error_code
|
|
||||||
```
|
|
||||||
|
|
||||||
Status:
|
|
||||||
|
|
||||||
```text
|
|
||||||
RUNNING
|
|
||||||
SUCCESS
|
|
||||||
PARTIAL
|
|
||||||
FAILED
|
|
||||||
BLOCKED
|
|
||||||
```
|
|
||||||
|
|
||||||
Es wird kein HTML-Snapshot gespeichert.
|
|
||||||
|
|
||||||
## 11. SourceProductReference
|
|
||||||
|
|
||||||
Ein Treffer einer externen Quelle wird als `SourceProductReference` dauerhaft angelegt.
|
|
||||||
|
|
||||||
Mindestens verfügbare Felder:
|
|
||||||
|
|
||||||
```text
|
|
||||||
id
|
|
||||||
source_id
|
|
||||||
source_category_id
|
|
||||||
source_url
|
|
||||||
source_title
|
|
||||||
manufacturer
|
|
||||||
model
|
|
||||||
manufacturer_sku
|
|
||||||
gtin_ean
|
|
||||||
normalized_spec_fingerprint
|
|
||||||
first_seen_at
|
|
||||||
last_seen_at
|
|
||||||
status
|
|
||||||
```
|
|
||||||
|
|
||||||
Keines dieser Felder außer der internen UUID wird pauschal als garantiert stabil betrachtet.
|
|
||||||
|
|
||||||
## 12. Produktidentität – Pilotansatz
|
|
||||||
|
|
||||||
v0.1 implementiert bewusst keine komplexe globale Identity Engine.
|
|
||||||
|
|
||||||
Für den Notebook-Pilot werden folgende Signale verwendet:
|
|
||||||
|
|
||||||
1. vorhandene stabile Source-ID, falls entdeckt
|
|
||||||
2. GTIN/EAN, falls vorhanden
|
|
||||||
3. Hersteller-SKU, falls vorhanden
|
|
||||||
4. kanonische Source-URL
|
|
||||||
5. normalisierter Titel
|
|
||||||
6. Hersteller + Modell
|
|
||||||
7. differenzierende technische Specs
|
|
||||||
|
|
||||||
Für Notebooks kann der initiale Fingerprint beispielsweise enthalten:
|
|
||||||
|
|
||||||
```text
|
|
||||||
manufacturer
|
|
||||||
model
|
|
||||||
cpu
|
|
||||||
ram
|
|
||||||
storage
|
|
||||||
display_size
|
|
||||||
gpu
|
|
||||||
```
|
|
||||||
|
|
||||||
### Matching-Regeln
|
|
||||||
|
|
||||||
- Hohe Sicherheit → existierende SourceProductReference verwenden
|
|
||||||
- Unsicherheit → `MATCH_UNCERTAIN`
|
|
||||||
- Keine sichere Übereinstimmung → neue SourceProductReference
|
|
||||||
- niemals zwei Produkte allein aufgrund ähnlicher Namen automatisch verschmelzen
|
|
||||||
|
|
||||||
Die tatsächliche Stabilität der Identitätsmerkmale wird im Notebook-Pilot beobachtet und danach verfeinert.
|
|
||||||
|
|
||||||
## 13. Produktbeobachtungen und Historie
|
|
||||||
|
|
||||||
Technische Eigenschaften und Preise werden nicht einfach überschrieben.
|
|
||||||
|
|
||||||
Jeder SearchRun kann neue Beobachtungen erzeugen.
|
|
||||||
|
|
||||||
### ProductObservation
|
|
||||||
|
|
||||||
```text
|
|
||||||
source_product_reference_id
|
|
||||||
search_run_id
|
|
||||||
observed_at
|
|
||||||
raw_title
|
|
||||||
best_price
|
|
||||||
currency
|
|
||||||
raw_attributes
|
|
||||||
normalized_attributes
|
|
||||||
availability
|
|
||||||
seller_count
|
|
||||||
```
|
|
||||||
|
|
||||||
Veränderte Werte bleiben historisch nachvollziehbar.
|
|
||||||
|
|
||||||
Beispiele:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Preis: 1499 → 1299
|
|
||||||
RAM: 16 → 32
|
|
||||||
Anbieterzahl: 8 → 12
|
|
||||||
Verfügbarkeit geändert
|
|
||||||
```
|
|
||||||
|
|
||||||
Bei technischen Attributänderungen muss das System berücksichtigen, dass auch fehlerhaftes Matching vorliegen könnte.
|
|
||||||
|
|
||||||
## 14. Diffing
|
|
||||||
|
|
||||||
Jeder erfolgreiche Full SearchRun wird mit dem vorherigen erfolgreichen Full SearchRun derselben DiscoverySearch verglichen.
|
|
||||||
|
|
||||||
### Status
|
|
||||||
|
|
||||||
```text
|
|
||||||
NEW
|
|
||||||
CHANGED
|
|
||||||
UNCHANGED
|
|
||||||
MISSING
|
|
||||||
DISAPPEARED
|
|
||||||
```
|
|
||||||
|
|
||||||
### Regeln
|
|
||||||
|
|
||||||
**NEW**
|
|
||||||
Produkt war im vorherigen vollständigen Lauf nicht vorhanden.
|
|
||||||
|
|
||||||
**CHANGED**
|
|
||||||
Produkt vorhanden, aber mindestens ein relevantes beobachtetes Feld hat sich verändert.
|
|
||||||
|
|
||||||
**UNCHANGED**
|
|
||||||
Produkt vorhanden und relevante Felder unverändert.
|
|
||||||
|
|
||||||
**MISSING**
|
|
||||||
Produkt fehlt in einem vollständigen Lauf.
|
|
||||||
|
|
||||||
**DISAPPEARED**
|
|
||||||
Produkt fehlt in drei aufeinanderfolgenden vollständigen Läufen.
|
|
||||||
|
|
||||||
Produkte werden nicht gelöscht.
|
|
||||||
|
|
||||||
Historie bleibt erhalten.
|
|
||||||
|
|
||||||
Ein später erneut auftauchendes Produkt wird reaktiviert und nicht als neues Produkt dupliziert, sofern das Matching ausreichend sicher ist.
|
|
||||||
|
|
||||||
## 15. Lifecycle eines BuyingProjects
|
|
||||||
|
|
||||||
Während ein BuyingProject aktiv ist:
|
|
||||||
|
|
||||||
```text
|
|
||||||
ACTIVE
|
|
||||||
↓
|
|
||||||
DiscoverySearches laufen
|
|
||||||
↓
|
|
||||||
Produkte + Historie wachsen
|
|
||||||
↓
|
|
||||||
Ranking + Shortlist
|
|
||||||
```
|
|
||||||
|
|
||||||
Nach finalem Kauf:
|
|
||||||
|
|
||||||
```text
|
|
||||||
PURCHASED
|
|
||||||
↓
|
|
||||||
BuyingProject archivieren
|
|
||||||
↓
|
|
||||||
Discovery stoppen
|
|
||||||
```
|
|
||||||
|
|
||||||
Produkte, Beobachtungen und Evaluationen können mit dem Projekt archiviert bzw. aus aktiven Ansichten entfernt werden.
|
|
||||||
|
|
||||||
Keine physische Löschung historischer Daten als Standardverhalten.
|
|
||||||
|
|
||||||
## 16. Ranking
|
|
||||||
|
|
||||||
Nach Discovery übernimmt ausschließlich die interne Buying Engine.
|
|
||||||
|
|
||||||
```text
|
|
||||||
Source Results
|
|
||||||
↓
|
|
||||||
interne Hard Constraints
|
|
||||||
↓
|
|
||||||
ProductScore
|
|
||||||
↓
|
|
||||||
Ranking
|
|
||||||
```
|
|
||||||
|
|
||||||
Geizhals-Sortierung darf für die technische Ausgabe genutzt werden, ist aber keine fachliche Ranking-Entscheidung.
|
|
||||||
|
|
||||||
## 17. Shortlist
|
|
||||||
|
|
||||||
Nach internem Ranking wird eine Shortlist gebildet.
|
|
||||||
|
|
||||||
Default:
|
|
||||||
|
|
||||||
```text
|
|
||||||
shortlist_size = 5
|
|
||||||
```
|
|
||||||
|
|
||||||
Der Wert ist pro BuyingProject konfigurierbar.
|
|
||||||
|
|
||||||
Shortlist-Kandidaten erhalten:
|
|
||||||
|
|
||||||
```text
|
|
||||||
SHORTLISTED
|
|
||||||
```
|
|
||||||
|
|
||||||
Andere gültige Kandidaten:
|
|
||||||
|
|
||||||
```text
|
|
||||||
CANDIDATE
|
|
||||||
```
|
|
||||||
|
|
||||||
Die spätere Deep-Enrichment-Stufe arbeitet primär auf der Shortlist.
|
|
||||||
|
|
||||||
## 18. Deep Enrichment
|
|
||||||
|
|
||||||
Nicht Bestandteil dieses Implementierungsslices.
|
|
||||||
|
|
||||||
Zielbild:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Shortlist
|
|
||||||
↓
|
|
||||||
Herstellerdaten
|
|
||||||
Tests
|
|
||||||
Benchmarks
|
|
||||||
Reviews
|
|
||||||
weitere Händler
|
|
||||||
Preisportale
|
|
||||||
Qualitätsinformationen
|
|
||||||
↓
|
|
||||||
erneute Bewertung
|
|
||||||
```
|
|
||||||
|
|
||||||
Discovery darf daher nicht voraussetzen, dass Geizhals alle für die finale Kaufentscheidung notwendigen Daten liefert.
|
|
||||||
|
|
||||||
## 19. Discovery, Ranking und Decision
|
|
||||||
|
|
||||||
Die drei fachlichen Stufen müssen getrennt bleiben.
|
|
||||||
|
|
||||||
### Discovery
|
|
||||||
|
|
||||||
```text
|
|
||||||
Welche Produkte kommen potenziell infrage?
|
|
||||||
```
|
|
||||||
|
|
||||||
### Ranking
|
|
||||||
|
|
||||||
```text
|
|
||||||
Welche dieser Produkte passen am besten zum BuyingProject?
|
|
||||||
```
|
|
||||||
|
|
||||||
### Decision
|
|
||||||
|
|
||||||
```text
|
|
||||||
Ist jetzt der richtige Zeitpunkt, Kandidat X zu kaufen?
|
|
||||||
```
|
|
||||||
|
|
||||||
Decision wird später u. a. berücksichtigen können:
|
|
||||||
|
|
||||||
- Zielpreis
|
|
||||||
- Deadline
|
|
||||||
- historische Preisentwicklung
|
|
||||||
- aktuelle Verfügbarkeit
|
|
||||||
- Product Fit
|
|
||||||
- Offer Fit
|
|
||||||
- Alternativen
|
|
||||||
- Beschaffungsdringlichkeit
|
|
||||||
|
|
||||||
Decision ist nicht Bestandteil dieses Slices.
|
|
||||||
|
|
||||||
## 20. Scheduling und defensiver Abruf
|
|
||||||
|
|
||||||
Default:
|
|
||||||
|
|
||||||
```text
|
|
||||||
alle 180 Minuten
|
|
||||||
```
|
|
||||||
|
|
||||||
pro DiscoverySearch konfigurierbar.
|
|
||||||
|
|
||||||
### Anforderungen
|
|
||||||
|
|
||||||
- keine parallelen Massenabrufe
|
|
||||||
- Pagination sequenziell oder stark begrenzt parallel
|
|
||||||
- Caching, wo sinnvoll
|
|
||||||
- Timeout
|
|
||||||
- Retry mit exponentiellem Backoff
|
|
||||||
- Jitter
|
|
||||||
- Stop bei wiederholten Fehlern
|
|
||||||
- keine Umgehung von Captchas oder Bot-Schutz
|
|
||||||
|
|
||||||
### Reaktion
|
|
||||||
|
|
||||||
**HTTP 429**
|
|
||||||
|
|
||||||
```text
|
|
||||||
BACKOFF
|
|
||||||
```
|
|
||||||
|
|
||||||
**HTTP 403**
|
|
||||||
|
|
||||||
```text
|
|
||||||
BLOCKED
|
|
||||||
```
|
|
||||||
|
|
||||||
**Captcha / Challenge erkannt**
|
|
||||||
|
|
||||||
```text
|
|
||||||
BLOCKED
|
|
||||||
```
|
|
||||||
|
|
||||||
Automatische weitere Abrufe der betroffenen DiscoverySearch werden pausiert.
|
|
||||||
|
|
||||||
Keine Schutzmechanismen umgehen.
|
|
||||||
|
|
||||||
## 21. Source Adapter Contract
|
|
||||||
|
|
||||||
Ein Discovery-Adapter soll mindestens folgende Fähigkeiten kapseln:
|
|
||||||
|
|
||||||
```python
|
|
||||||
class DiscoverySourceAdapter(Protocol):
|
|
||||||
def discover_taxonomy(category_ref): ...
|
|
||||||
def build_query(criteria, mappings): ...
|
|
||||||
def fetch_results(query, pagination): ...
|
|
||||||
def parse_results(response): ...
|
|
||||||
def normalize_result(result): ...
|
|
||||||
```
|
|
||||||
|
|
||||||
Taxonomy Learning und Result Retrieval dürfen intern getrennte Services sein.
|
|
||||||
|
|
||||||
Der Rest des Systems darf keine Geizhals-spezifische URL- oder HTML-Logik kennen.
|
|
||||||
|
|
||||||
## 22. Services
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
```text
|
|
||||||
TaxonomyService
|
|
||||||
CriterionMappingService
|
|
||||||
DiscoverySearchService
|
|
||||||
SearchExecutionService
|
|
||||||
SourceProductIdentityService
|
|
||||||
SearchDiffService
|
|
||||||
CandidateRankingService
|
|
||||||
ShortlistService
|
|
||||||
```
|
|
||||||
|
|
||||||
Keine Businesslogik in n8n.
|
|
||||||
|
|
||||||
n8n kann später den Scheduler bzw. externe Events orchestrieren.
|
|
||||||
|
|
||||||
## 23. Persistenz
|
|
||||||
|
|
||||||
Neue bzw. ergänzende Tabellen können sein:
|
|
||||||
|
|
||||||
```text
|
|
||||||
source_categories
|
|
||||||
source_filter_definitions
|
|
||||||
source_filter_options
|
|
||||||
source_filter_ranges
|
|
||||||
criterion_definitions
|
|
||||||
criterion_mappings
|
|
||||||
discovery_searches
|
|
||||||
search_runs
|
|
||||||
source_product_references
|
|
||||||
product_observations
|
|
||||||
project_candidates
|
|
||||||
candidate_status_history
|
|
||||||
```
|
|
||||||
|
|
||||||
Bestehende globale Entitäten aus der Hauptspezifikation bleiben bestehen:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Product
|
|
||||||
ProductVariant
|
|
||||||
Offer
|
|
||||||
PriceObservation
|
|
||||||
```
|
|
||||||
|
|
||||||
Diese dürfen nicht unnötig dupliziert werden.
|
|
||||||
|
|
||||||
Der Discovery-Slice kann zunächst mit SourceProductReference arbeiten und erst bei ausreichender Identität eine Verknüpfung zu canonical Product/ProductVariant herstellen.
|
|
||||||
|
|
||||||
## 24. API – minimaler Vertrag
|
|
||||||
|
|
||||||
### Taxonomie
|
|
||||||
|
|
||||||
```text
|
|
||||||
POST /api/v1/sources/geizhals/categories/{category}/discover-taxonomy
|
|
||||||
GET /api/v1/sources/geizhals/categories/{category}/filters
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mapping
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/criterion-mappings
|
|
||||||
POST /api/v1/criterion-mappings
|
|
||||||
PATCH /api/v1/criterion-mappings/{id}
|
|
||||||
```
|
|
||||||
|
|
||||||
### DiscoverySearch
|
|
||||||
|
|
||||||
```text
|
|
||||||
POST /api/v1/discovery-searches
|
|
||||||
GET /api/v1/discovery-searches/{id}
|
|
||||||
POST /api/v1/discovery-searches/{id}/generate-query
|
|
||||||
POST /api/v1/discovery-searches/{id}/run
|
|
||||||
```
|
|
||||||
|
|
||||||
### SearchRuns
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/discovery-searches/{id}/runs
|
|
||||||
GET /api/v1/search-runs/{id}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Candidates
|
|
||||||
|
|
||||||
```text
|
|
||||||
GET /api/v1/buying-projects/{id}/candidates
|
|
||||||
GET /api/v1/buying-projects/{id}/shortlist
|
|
||||||
```
|
|
||||||
|
|
||||||
Falls BuyingProject im aktuellen Code noch nicht existiert, muss die Integration mit der im Architecture Addendum vorgesehenen nächsten Domain-Erweiterung abgestimmt werden.
|
|
||||||
|
|
||||||
## 25. Tests
|
|
||||||
|
|
||||||
### Taxonomy
|
|
||||||
|
|
||||||
- Notebook-Kategorie kann on demand eingelesen werden
|
|
||||||
- Filterdefinitionen werden reproduzierbar gespeichert
|
|
||||||
- Optionen werden nicht dupliziert
|
|
||||||
- Filteränderungen werden erkannt
|
|
||||||
- BROKEN-Mapping wird nicht produktiv verwendet
|
|
||||||
|
|
||||||
### Query Generation
|
|
||||||
|
|
||||||
- identische Criteria + Mapping-Version erzeugen identische URL
|
|
||||||
- RAM >= 24 erzeugt erwartetes Geizhals-Fragment
|
|
||||||
- Preisbereich korrekt
|
|
||||||
- mehrere Filter kombinierbar
|
|
||||||
- INTERNAL_ONLY-Kriterien verändern die Source-URL nicht
|
|
||||||
- veraltete Mapping-Version markiert Search als OUT_OF_SYNC
|
|
||||||
|
|
||||||
### Search Execution
|
|
||||||
|
|
||||||
- Pagination vollständig
|
|
||||||
- Ergebniszahl korrekt
|
|
||||||
- TOO_BROAD bei Überschreitung
|
|
||||||
- kein stilles Top-N-Abschneiden
|
|
||||||
- 429 → Backoff
|
|
||||||
- 403/Captcha → BLOCKED
|
|
||||||
|
|
||||||
### Identity
|
|
||||||
|
|
||||||
- identische SKU wird wiedererkannt
|
|
||||||
- identische EAN wird wiedererkannt
|
|
||||||
- gleiche URL wird wiedererkannt
|
|
||||||
- leicht geänderter Titel bei stabilen Specs wird erkannt
|
|
||||||
- unsicheres Match erzeugt MATCH_UNCERTAIN
|
|
||||||
- ähnliche Produktnamen werden nicht automatisch verschmolzen
|
|
||||||
|
|
||||||
### Diffing
|
|
||||||
|
|
||||||
- neuer Treffer → NEW
|
|
||||||
- geänderter Preis → CHANGED
|
|
||||||
- unverändert → UNCHANGED
|
|
||||||
- einmal fehlend → MISSING
|
|
||||||
- dreimal fehlend → DISAPPEARED
|
|
||||||
- Wiederauftauchen reaktiviert bestehende Referenz
|
|
||||||
|
|
||||||
### Historie
|
|
||||||
|
|
||||||
- Preisänderungen bleiben historisiert
|
|
||||||
- technische Änderungen bleiben historisiert
|
|
||||||
- alte SearchRuns bleiben unverändert
|
|
||||||
- Projektarchivierung löscht keine History
|
|
||||||
|
|
||||||
### Ranking
|
|
||||||
|
|
||||||
- nur interne Engine bestimmt Ranking
|
|
||||||
- Source-Sortierung beeinflusst ProductScore nicht
|
|
||||||
- Shortlistgröße Default 5
|
|
||||||
- Shortlistgröße konfigurierbar
|
|
||||||
|
|
||||||
## 26. Implementierungsreihenfolge
|
|
||||||
|
|
||||||
1. Bestehende Implementierung gegen diese Spec und das Architecture Addendum prüfen.
|
|
||||||
2. Notwendige BuyingProject-/ProjectCandidate-Minimalstruktur ergänzen, falls für Discovery erforderlich.
|
|
||||||
3. Source Taxonomy + Mapping persistieren.
|
|
||||||
4. Geizhals-Taxonomie-Pilot für Notebooks implementieren.
|
|
||||||
5. reproduzierbare Query Generation implementieren.
|
|
||||||
6. SearchRun + Pagination implementieren.
|
|
||||||
7. SourceProductReference + ProductObservation implementieren.
|
|
||||||
8. Identity Pilot implementieren.
|
|
||||||
9. Diffing implementieren.
|
|
||||||
10. interne Ranking-Engine anbinden.
|
|
||||||
11. Shortlist bilden.
|
|
||||||
12. Scheduling-Schnittstelle vorbereiten bzw. internen Scheduler minimal anbinden.
|
|
||||||
13. Tests vollständig ausführen.
|
|
||||||
14. Notebook-Pilot mit realer Suche validieren.
|
|
||||||
|
|
||||||
## 27. Definition of Done
|
|
||||||
|
|
||||||
- [ ] Geizhals-Notebook-Taxonomie wird on demand eingelesen und gespeichert.
|
|
||||||
- [ ] Source-Filter und Optionen werden persistiert.
|
|
||||||
- [ ] Filter-Mappings besitzen Lifecycle-Status.
|
|
||||||
- [ ] BuyingProject-Kriterien erzeugen reproduzierbar eine Geizhals-URL.
|
|
||||||
- [ ] Nicht abbildbare Kriterien bleiben INTERNAL_ONLY.
|
|
||||||
- [ ] Aktive Suche kann vollständig ausgeführt werden.
|
|
||||||
- [ ] Pagination wird vollständig verarbeitet.
|
|
||||||
- [ ] TOO_BROAD wird erkannt.
|
|
||||||
- [ ] SearchRuns werden gespeichert.
|
|
||||||
- [ ] Produkte werden dauerhaft als SourceProductReference angelegt.
|
|
||||||
- [ ] Preise und relevante Attribute werden historisiert.
|
|
||||||
- [ ] NEW / CHANGED / UNCHANGED / MISSING / DISAPPEARED funktionieren.
|
|
||||||
- [ ] DISAPPEARED erst nach drei aufeinanderfolgenden fehlenden Full Runs.
|
|
||||||
- [ ] Unsichere Produktidentität wird nicht automatisch verschmolzen.
|
|
||||||
- [ ] interne Engine bewertet Kandidaten.
|
|
||||||
- [ ] Shortlist Default 5 funktioniert.
|
|
||||||
- [ ] Suchfrequenz ist konfigurierbar, Default 180 Minuten.
|
|
||||||
- [ ] 403/429/Captcha führen zu defensivem Stop/Backoff.
|
|
||||||
- [ ] keine HTML-Archive.
|
|
||||||
- [ ] keine Notebook-spezifische Businesslogik außerhalb des Geizhals-Pilotmappings.
|
|
||||||
- [ ] Tests grün.
|
|
||||||
- [ ] Ergebnisse des Notebook-Piloten dokumentiert.
|
|
||||||
|
|
||||||
# Coding-Agent-Arbeitsauftrag
|
|
||||||
|
|
||||||
Verwende als verbindliche Anforderungen:
|
|
||||||
|
|
||||||
1. `buying-agent-v0.1-spec-v1.1.md`
|
|
||||||
2. `buying-agent-architecture-addendum-v1.md`
|
|
||||||
3. `buying-agent-source-discovery-spec-v0.1.md`
|
|
||||||
|
|
||||||
Die neue Source-/Discovery-Spec ergänzt die bestehenden Dokumente. Bei Konflikten in ihrem Fachbereich hat sie Vorrang.
|
|
||||||
|
|
||||||
## Vorgehen
|
|
||||||
|
|
||||||
### 1. Zuerst analysieren
|
|
||||||
|
|
||||||
Prüfe den aktuellen Repository-Stand gegen alle drei Dokumente.
|
|
||||||
|
|
||||||
Berichte vor größeren Änderungen:
|
|
||||||
|
|
||||||
- welche Teile bereits kompatibel sind,
|
|
||||||
- welche Domain-/DB-/API-Erweiterungen erforderlich sind,
|
|
||||||
- ob bestehende Implementierung refaktoriert werden muss,
|
|
||||||
- welche Änderungen rein additiv möglich sind.
|
|
||||||
|
|
||||||
### 2. Minimal integrieren
|
|
||||||
|
|
||||||
Implementiere nur den beschriebenen Source-/Discovery-Slice.
|
|
||||||
|
|
||||||
Nicht implementieren:
|
|
||||||
|
|
||||||
- Frontend
|
|
||||||
- Keepa
|
|
||||||
- Idealo
|
|
||||||
- LLM-Automation
|
|
||||||
- Auth/RBAC
|
|
||||||
- Kaufautomatisierung
|
|
||||||
- Warenkorb
|
|
||||||
|
|
||||||
### 3. Besonders beachten
|
|
||||||
|
|
||||||
- nicht notebook-spezifisch modellieren
|
|
||||||
- Geizhals-spezifische Logik ausschließlich im Source Adapter / Mapping Layer
|
|
||||||
- URL-Encoding nur aus verifizierten Source-Mappings
|
|
||||||
- vollständige SearchRuns, kein stilles Top-N-Limit
|
|
||||||
- konservatives Product Matching
|
|
||||||
- keine automatische Zusammenführung bei Unsicherheit
|
|
||||||
- alle Änderungen historisieren
|
|
||||||
- Discovery, Ranking und Decision getrennt halten
|
|
||||||
|
|
||||||
### 4. Tests
|
|
||||||
|
|
||||||
Führe Unit-, Integration- und End-to-End-Tests aus.
|
|
||||||
|
|
||||||
Mindestens:
|
|
||||||
|
|
||||||
- Taxonomy Discovery
|
|
||||||
- Query Generation
|
|
||||||
- Pagination
|
|
||||||
- Identity Matching
|
|
||||||
- Diffing
|
|
||||||
- History
|
|
||||||
- Ranking
|
|
||||||
- Shortlist
|
|
||||||
- Backoff / BLOCKED
|
|
||||||
|
|
||||||
### 5. Abschlussbericht
|
|
||||||
|
|
||||||
Dokumentiere:
|
|
||||||
|
|
||||||
- geänderte Dateien
|
|
||||||
- neue Migrationen
|
|
||||||
- neue API-Endpunkte
|
|
||||||
- Testergebnisse
|
|
||||||
- real durchgeführten Notebook-Pilot
|
|
||||||
- erkannte Geizhals-Eigenheiten
|
|
||||||
- verbleibende Risiken
|
|
||||||
- insbesondere Beobachtungen zur Stabilität der Produktidentität
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,9 +1,11 @@
|
||||||
FROM ghcr.io/astral-sh/uv:0.10.11-python3.13-trixie-slim AS base
|
FROM python:3.13-slim-bookworm AS base
|
||||||
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends libpq5 \
|
&& apt-get install -y --no-install-recommends libpq5 \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=ghcr.io/astral-sh/uv:0.10.11 /uv /usr/local/bin/uv
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
ENV UV_COMPILE_BYTECODE=1 \
|
ENV UV_COMPILE_BYTECODE=1 \
|
||||||
|
|
@ -18,13 +20,11 @@ RUN uv sync --frozen --no-dev --no-install-project
|
||||||
FROM base AS runtime
|
FROM base AS runtime
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
|
RUN uv sync --frozen --no-dev \
|
||||||
RUN uv sync --frozen --extra live --extra test --no-dev \
|
|
||||||
&& playwright install --with-deps chromium \
|
|
||||||
&& sed -i 's/\r$//' /app/scripts/startup.sh \
|
&& sed -i 's/\r$//' /app/scripts/startup.sh \
|
||||||
&& chmod +x /app/scripts/startup.sh \
|
&& chmod +x /app/scripts/startup.sh \
|
||||||
&& useradd --create-home --uid 1000 app \
|
&& useradd --create-home --uid 1000 app \
|
||||||
&& chown -R app:app /app /ms-playwright
|
&& chown -R app:app /app
|
||||||
|
|
||||||
USER app
|
USER app
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
|
|
@ -39,4 +39,4 @@ RUN uv sync --frozen --extra test \
|
||||||
&& chown -R app:app /app
|
&& chown -R app:app /app
|
||||||
|
|
||||||
USER app
|
USER app
|
||||||
CMD ["pytest", "tests", "-m", "not slow", "-ra", "--tb=short"]
|
CMD ["pytest", "tests", "-ra"]
|
||||||
67
README.md
67
README.md
|
|
@ -1,95 +1,42 @@
|
||||||
# BuyingAgent
|
# BuyingAgent
|
||||||
|
|
||||||
Backend für einen generischen Buying-/Deal-Agenten. Das System bewertet, wie gut eine Produktvariante zu einem Profil passt und wie attraktiv ein konkretes Angebot preislich ist. Es kauft nicht.
|
Backend für einen generischen Buying-/Deal-Agenten. Der erste fachliche Schnitt ist in `.specs/buying-agent-v0.1-spec.md` beschrieben.
|
||||||
|
|
||||||
v0.1 ist ein Walking Skeleton: Profil → Produkt → Variante → Angebot → Preisbeobachtungen → Evaluation. Notebooks sind der erste Anwendungsfall, nicht das Datenmodell.
|
Entwicklung auf `develop`. Push nach `develop` baut die Dev-Umgebung auf dem Raspberry Pi. Ein Merge-Commit nach `main` deployt Production. Details: `docs/DEPLOYMENT.md`.
|
||||||
|
|
||||||
**Vorgaben:** `docs/buying-agent-v0.1-spec-v1.1.md` (fachlich) und `docs/buying-agent-architecture-addendum-v1.md` (Architektur, hat Vorrang bei Strukturfragen).
|
|
||||||
|
|
||||||
## Monorepo
|
|
||||||
|
|
||||||
```text
|
|
||||||
backend/ System of Record: Domain, Scoring, Persistenz, /api/v1
|
|
||||||
frontend/ Platzhalter — keine UI in v0.1
|
|
||||||
docs/ Spezifikationen und Deployment
|
|
||||||
docker-compose.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Entwicklung auf `develop`. Push nach `develop` baut Dev auf dem Raspberry Pi. Merge `--no-ff` nach `main` deployt Production. Details: `docs/DEPLOYMENT.md`.
|
|
||||||
|
|
||||||
## Stack
|
## Stack
|
||||||
|
|
||||||
Python 3.13, FastAPI, Pydantic 2, SQLAlchemy 2 (sync, psycopg 3), Alembic, PostgreSQL 17, pytest, Docker Compose. Build-Tool: uv 0.10.11.
|
Python 3.13, FastAPI, Pydantic 2, SQLAlchemy 2, Alembic, PostgreSQL 17, pytest, Docker Compose. Build-Tool: [uv](https://docs.astral.sh/uv/) 0.10.11.
|
||||||
|
|
||||||
Abhängigkeiten: HTTP → Services → Domain/Scoring → Repositories → PostgreSQL. `domain` und `scoring` importieren weder FastAPI noch SQLAlchemy.
|
|
||||||
|
|
||||||
## Lokal starten
|
## Lokal starten
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
Copy-Item .env.example .env
|
Copy-Item .env.example .env
|
||||||
docker compose up --build -d
|
docker compose up --build -d
|
||||||
docker compose exec api python -m scripts.demo
|
|
||||||
```
|
```
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
docker compose up --build -d
|
docker compose up --build -d
|
||||||
docker compose exec api python -m scripts.demo
|
|
||||||
```
|
```
|
||||||
|
|
||||||
API: `http://127.0.0.1:8000`
|
API: `http://127.0.0.1:8000`
|
||||||
OpenAPI: `http://127.0.0.1:8000/docs`
|
|
||||||
Live: `GET /health/live`
|
Live: `GET /health/live`
|
||||||
Ready: `GET /health/ready`
|
Ready: `GET /health/ready` (PostgreSQL erreichbar und Alembic auf Head)
|
||||||
|
|
||||||
Die Demo legt Definitionen, Profil und die Fälle A–D über HTTP an und wertet sie mit `as_of=2026-10-03T12:00:00Z` aus. Wiederholter Lauf erzeugt keine Duplikate.
|
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
Lokal, isolierter Stack (tmpfs, kein Dev-/Prod-Volume):
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from api
|
docker compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from api
|
||||||
```
|
```
|
||||||
|
|
||||||
Nur Unit-Tests (ohne PostgreSQL), im Verzeichnis `backend/`:
|
Die Testdatenbank liegt auf tmpfs und leert kein normales Datenvolume.
|
||||||
|
|
||||||
```sh
|
|
||||||
pytest tests/unit -ra --tb=short
|
|
||||||
```
|
|
||||||
|
|
||||||
Auf dem Server nach Dev-Deploy (Gitea `test.yml`): pytest gegen `buyingagent_test` auf dem Dev-Postgres plus Smoke gegen `http://192.168.2.49:8095/health/ready`.
|
|
||||||
|
|
||||||
## Datenmodell (global)
|
|
||||||
|
|
||||||
Product, ProductVariant, Offer, PriceObservation, AttributeDefinition und Source sind **systemweit** und haben keine `user_id`/`workspace_id`. Workspace, BuyingProject und Benutzer kommen später und dürfen diese Daten nicht duplizieren.
|
|
||||||
|
|
||||||
Profile und Evaluationen können bereits parallel existieren; es gibt kein globales „aktuelles Kaufvorhaben“.
|
|
||||||
|
|
||||||
## Scoring `generic-v1`
|
|
||||||
|
|
||||||
- Preference-Strategie nur `numeric_linear`. Unbekannte Strategien werden beim Profil abgewiesen.
|
|
||||||
- `product_eligibility`, `offer_eligibility` und `overall_eligibility` sind getrennt.
|
|
||||||
- PriceScore ist eine reine Funktion über `current_price`, `reference_history` und `budget`. v0.1 setzt `price_reference_scope=offer`.
|
|
||||||
- DealScore nur bei `overall_eligibility=ELIGIBLE` und vorhandenem PriceScore.
|
|
||||||
- Klassifikation am **ungerundeten** DealScore; Anzeige auf zwei Dezimalen, `ROUND_HALF_UP`.
|
|
||||||
- Geld intern `Decimal`/`NUMERIC(12,2)`, in der API als Dezimalstring.
|
|
||||||
|
|
||||||
Referenzfall A (synthetisch): ProductScore 90.21, PriceScore 98.24, DealScore 93.42, `EXCEPTIONAL`. Das ist keine Marktaussage.
|
|
||||||
|
|
||||||
## Fehler
|
|
||||||
|
|
||||||
Einheitliches Format `{error: {code, message, details}}`. 404 fehlende ID, 409 Konflikt/abweichender Wiederholungsimport, 422 ungültige Payload. Eine fachlich negative Evaluation ist HTTP 201.
|
|
||||||
|
|
||||||
## Grenzen von v0.1
|
|
||||||
|
|
||||||
Keine Web-UI, keine Authentifizierung, keine Workspaces, kein BuyingProject, kein n8n, keine externen Quellen, keine Auto-Discovery, keine Wechselkurse, keine Variant-/Market-Level-Preisaggregation, keine Plugin-Infrastruktur.
|
|
||||||
|
|
||||||
## Server
|
## Server
|
||||||
|
|
||||||
| Umgebung | Branch | Verzeichnis | API |
|
| Umgebung | Branch | Verzeichnis | API |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| Development | `develop` | `~/docker/buyingagent-dev` | `http://192.168.2.49:8095` |
|
| Development | `develop` | `/home/lars/docker/buyingagent-dev` | `http://192.168.2.49:8095` |
|
||||||
| Production | `main` | `~/docker/buyingagent` | `http://192.168.2.49:8006` |
|
| Production | `main` | `/home/lars/docker/buyingagent` | `http://192.168.2.49:8006` |
|
||||||
|
|
||||||
Repo: `http://192.168.2.144:3000/Lars/BuyingAgent.git`
|
Repo: `http://192.168.2.144:3000/Lars/BuyingAgent.git`
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
[alembic]
|
[alembic]
|
||||||
script_location = migrations
|
script_location = migrations
|
||||||
prepend_sys_path = .
|
prepend_sys_path = .
|
||||||
path_separator = os
|
|
||||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||||
|
|
||||||
[loggers]
|
[loggers]
|
||||||
1
app/__init__.py
Normal file
1
app/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
__version__ = "0.1.0"
|
||||||
10
app/main.py
Normal file
10
app/main.py
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
from fastapi import FastAPI
|
||||||
|
|
||||||
|
from app.api.health import router as health_router
|
||||||
|
|
||||||
|
app = FastAPI(
|
||||||
|
title="Buying Agent",
|
||||||
|
version="0.1.0",
|
||||||
|
description="Walking skeleton for the buying and deal evaluation API.",
|
||||||
|
)
|
||||||
|
app.include_router(health_router)
|
||||||
|
|
@ -14,9 +14,5 @@ def get_session() -> Generator[Session, None, None]:
|
||||||
session = SessionLocal()
|
session = SessionLocal()
|
||||||
try:
|
try:
|
||||||
yield session
|
yield session
|
||||||
session.commit()
|
|
||||||
except Exception:
|
|
||||||
session.rollback()
|
|
||||||
raise
|
|
||||||
finally:
|
finally:
|
||||||
session.close()
|
session.close()
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
from datetime import datetime
|
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
@ -14,14 +13,6 @@ class Settings(BaseSettings):
|
||||||
db_password: str = "change_me"
|
db_password: str = "change_me"
|
||||||
environment: str = "development"
|
environment: str = "development"
|
||||||
engine_build: str = "dev"
|
engine_build: str = "dev"
|
||||||
clock_now: datetime | None = None
|
|
||||||
cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173"
|
|
||||||
geizhals_use_fixtures: bool = False
|
|
||||||
geizhals_playwright: bool = True
|
|
||||||
|
|
||||||
@property
|
|
||||||
def cors_origins(self) -> list[str]:
|
|
||||||
return [item.strip() for item in self.cors_allowed_origins.split(",") if item.strip()]
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def database_url(self) -> str:
|
def database_url(self) -> str:
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
.git
|
|
||||||
.venv
|
|
||||||
.pytest_cache
|
|
||||||
.mypy_cache
|
|
||||||
.ruff_cache
|
|
||||||
__pycache__
|
|
||||||
*.pyc
|
|
||||||
*.pyo
|
|
||||||
*.egg-info
|
|
||||||
.tests-tmp
|
|
||||||
|
|
@ -1,136 +0,0 @@
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.api.schemas import (
|
|
||||||
CriterionDefinitionIn,
|
|
||||||
CriterionMappingIn,
|
|
||||||
CriterionMappingPatch,
|
|
||||||
DiscoverySearchIn,
|
|
||||||
DiscoverySearchPatch,
|
|
||||||
SourceAttributeMappingIn,
|
|
||||||
SourceAttributeMappingPatch,
|
|
||||||
SourceCategoryIn,
|
|
||||||
)
|
|
||||||
from app.persistence.db import get_session
|
|
||||||
from app.services.catalog_service import CatalogService
|
|
||||||
from app.services.discovery_service import (
|
|
||||||
CriterionDefinitionService,
|
|
||||||
CriterionMappingService,
|
|
||||||
DiscoverySearchService,
|
|
||||||
SourceAttributeMappingService,
|
|
||||||
TaxonomyService,
|
|
||||||
)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1")
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/sources/{source_id}/categories")
|
|
||||||
def list_source_categories(source_id: str, session: Session = Depends(get_session)):
|
|
||||||
return {"items": CatalogService(session).source_categories(source_id)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/sources/{source_id}/categories", status_code=201)
|
|
||||||
def register_source_category(source_id: str, payload: SourceCategoryIn, session: Session = Depends(get_session)):
|
|
||||||
return CatalogService(session).register_source_category(source_id, payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/sources/{source_id}/categories/{category}/discover-taxonomy", status_code=201)
|
|
||||||
def discover_taxonomy(source_id: str, category: str, session: Session = Depends(get_session)):
|
|
||||||
return TaxonomyService(session).discover(source_id, category)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/sources/{source_id}/categories/{category}/taxonomy-status")
|
|
||||||
def taxonomy_status(source_id: str, category: str, session: Session = Depends(get_session)):
|
|
||||||
return TaxonomyService(session).taxonomy_status(source_id, category)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/sources/{source_id}/categories/{category}/filters")
|
|
||||||
def list_filters(source_id: str, category: str, session: Session = Depends(get_session)):
|
|
||||||
return TaxonomyService(session).filters(source_id, category)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/criterion-definitions", status_code=201)
|
|
||||||
def create_criterion_definition(payload: CriterionDefinitionIn, session: Session = Depends(get_session)):
|
|
||||||
return CriterionDefinitionService(session).create(payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/criterion-definitions")
|
|
||||||
def list_criterion_definitions(category: str | None = None, session: Session = Depends(get_session)):
|
|
||||||
return {"items": CriterionDefinitionService(session).list(category)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/criterion-mappings")
|
|
||||||
def list_mappings(source_category_id: UUID | None = None, session: Session = Depends(get_session)):
|
|
||||||
return {"items": CriterionMappingService(session).list(source_category_id)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/criterion-mappings", status_code=201)
|
|
||||||
def create_mapping(payload: CriterionMappingIn, session: Session = Depends(get_session)):
|
|
||||||
return CriterionMappingService(session).create(payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/criterion-mappings/{mapping_id}")
|
|
||||||
def patch_mapping(mapping_id: UUID, payload: CriterionMappingPatch, session: Session = Depends(get_session)):
|
|
||||||
return CriterionMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/source-attribute-mappings")
|
|
||||||
def list_attribute_mappings(session: Session = Depends(get_session)):
|
|
||||||
return {"items": SourceAttributeMappingService(session).list()}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/source-attribute-mappings", status_code=201)
|
|
||||||
def create_attribute_mapping(payload: SourceAttributeMappingIn, session: Session = Depends(get_session)):
|
|
||||||
return SourceAttributeMappingService(session).create(payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/source-attribute-mappings/{mapping_id}")
|
|
||||||
def patch_attribute_mapping(
|
|
||||||
mapping_id: UUID,
|
|
||||||
payload: SourceAttributeMappingPatch,
|
|
||||||
session: Session = Depends(get_session),
|
|
||||||
):
|
|
||||||
return SourceAttributeMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/discovery-searches", status_code=201)
|
|
||||||
def create_search(payload: DiscoverySearchIn, session: Session = Depends(get_session)):
|
|
||||||
return DiscoverySearchService(session).create(payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/discovery-searches")
|
|
||||||
def list_searches(due: bool = Query(default=False), session: Session = Depends(get_session)):
|
|
||||||
service = DiscoverySearchService(session)
|
|
||||||
return {"items": service.list_due() if due else service.list_all()}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/discovery-searches/{search_id}")
|
|
||||||
def get_search(search_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return DiscoverySearchService(session).get(search_id)
|
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/discovery-searches/{search_id}")
|
|
||||||
def patch_search(search_id: UUID, payload: DiscoverySearchPatch, session: Session = Depends(get_session)):
|
|
||||||
return DiscoverySearchService(session).patch(search_id, payload.model_dump(exclude_unset=True))
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/discovery-searches/{search_id}/generate-query")
|
|
||||||
def generate_query(search_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return DiscoverySearchService(session).generate_query(search_id)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/discovery-searches/{search_id}/run", status_code=201)
|
|
||||||
def run_search(search_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return DiscoverySearchService(session).run(search_id)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/discovery-searches/{search_id}/runs")
|
|
||||||
def list_runs(search_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return {"items": DiscoverySearchService(session).list_runs(search_id)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/search-runs/{run_id}")
|
|
||||||
def get_run(run_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return DiscoverySearchService(session).get_run(run_id)
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.api.schemas import EvaluateIn
|
|
||||||
from app.persistence.db import get_session
|
|
||||||
from app.services.evaluation_service import evaluate_and_store, load_evaluation
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/evaluate/{variant_id}", status_code=201)
|
|
||||||
def evaluate(variant_id: UUID, payload: EvaluateIn):
|
|
||||||
return evaluate_and_store(variant_id=variant_id, payload=payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/evaluations/{evaluation_id}")
|
|
||||||
def get_evaluation(evaluation_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return load_evaluation(session, evaluation_id)
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Response
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.api.schemas import OfferIn, OfferOut, PriceIn, PriceOut, page, parse_page
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.persistence.db import get_session
|
|
||||||
from app.persistence.repositories import CatalogRepository
|
|
||||||
from app.services.ingestion_service import IngestionService
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/variants/{variant_id}/offers", status_code=201)
|
|
||||||
def create_offer(variant_id: UUID, payload: OfferIn, session: Session = Depends(get_session)) -> OfferOut:
|
|
||||||
offer = IngestionService(session).create_offer(variant_id, payload.model_dump())
|
|
||||||
return OfferOut.model_validate(offer.__dict__)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/offers/{offer_id}")
|
|
||||||
def get_offer(offer_id: UUID, session: Session = Depends(get_session)) -> OfferOut:
|
|
||||||
offer = CatalogRepository(session).get_offer(offer_id)
|
|
||||||
if offer is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
|
||||||
return OfferOut.model_validate(offer.__dict__)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/offers/{offer_id}/prices")
|
|
||||||
def add_price(offer_id: UUID, payload: PriceIn, response: Response, session: Session = Depends(get_session)) -> PriceOut:
|
|
||||||
observation, created = IngestionService(session).add_price(offer_id, payload.model_dump())
|
|
||||||
response.status_code = 201 if created else 200
|
|
||||||
return PriceOut.model_validate(observation.__dict__)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/offers/{offer_id}/prices")
|
|
||||||
def list_prices(offer_id: UUID, limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
|
||||||
if CatalogRepository(session).get_offer(offer_id) is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
|
||||||
limit, offset = parse_page(limit, offset)
|
|
||||||
items, total = CatalogRepository(session).list_observations(offer_id, limit, offset)
|
|
||||||
return page([PriceOut.model_validate(item.__dict__) for item in items], limit, offset, total)
|
|
||||||
|
|
@ -1,63 +0,0 @@
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.api.schemas import AttributeIn, AttributeOut, ProductIn, ProductOut, VariantIn, VariantOut
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.persistence.db import get_session
|
|
||||||
from app.persistence.repositories import CatalogRepository
|
|
||||||
from app.services.ingestion_service import IngestionService
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/products", status_code=201)
|
|
||||||
def create_product(payload: ProductIn, session: Session = Depends(get_session)) -> ProductOut:
|
|
||||||
product = IngestionService(session).create_product(payload.model_dump())
|
|
||||||
return ProductOut.model_validate(product.__dict__)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/products/{product_id}")
|
|
||||||
def get_product(product_id: UUID, session: Session = Depends(get_session)) -> ProductOut:
|
|
||||||
product = CatalogRepository(session).get_product(product_id)
|
|
||||||
if product is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
|
|
||||||
return ProductOut.model_validate(product.__dict__)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/products/{product_id}/variants", status_code=201)
|
|
||||||
def create_variant(product_id: UUID, payload: VariantIn, session: Session = Depends(get_session)):
|
|
||||||
variant = IngestionService(session).create_variant(product_id, payload.model_dump())
|
|
||||||
return {
|
|
||||||
"id": variant.id,
|
|
||||||
"product_id": variant.product_id,
|
|
||||||
"name": variant.name,
|
|
||||||
"manufacturer_sku": variant.manufacturer_sku,
|
|
||||||
"identifiers": variant.identifiers,
|
|
||||||
"created_at": variant.created_at,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/variants/{variant_id}")
|
|
||||||
def get_variant(variant_id: UUID, session: Session = Depends(get_session)) -> VariantOut:
|
|
||||||
repo = CatalogRepository(session)
|
|
||||||
variant = repo.get_variant(variant_id)
|
|
||||||
if variant is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
|
||||||
attributes = [AttributeOut.model_validate(item.__dict__) for item in repo.list_attributes(variant_id)]
|
|
||||||
return VariantOut(
|
|
||||||
id=variant.id,
|
|
||||||
product_id=variant.product_id,
|
|
||||||
name=variant.name,
|
|
||||||
manufacturer_sku=variant.manufacturer_sku,
|
|
||||||
identifiers=variant.identifiers,
|
|
||||||
created_at=variant.created_at,
|
|
||||||
attributes=attributes,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/variants/{variant_id}/attributes", status_code=201)
|
|
||||||
def add_attribute(variant_id: UUID, payload: AttributeIn, session: Session = Depends(get_session)) -> AttributeOut:
|
|
||||||
statement = IngestionService(session).add_attribute(variant_id, payload.model_dump())
|
|
||||||
return AttributeOut.model_validate(statement.__dict__)
|
|
||||||
|
|
@ -1,100 +0,0 @@
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Depends
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.api.schemas import AttributeDefinitionIn, page, parse_page
|
|
||||||
from app.domain.serialize import jsonable
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.persistence.db import get_session
|
|
||||||
from app.persistence.repositories import CatalogRepository
|
|
||||||
from app.services.ingestion_service import IngestionService
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/attribute-definitions", status_code=201)
|
|
||||||
def create_attribute_definition(payload: AttributeDefinitionIn, session: Session = Depends(get_session)):
|
|
||||||
definition = IngestionService(session).create_definition(payload.model_dump())
|
|
||||||
return jsonable(definition.__dict__)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/attribute-definitions")
|
|
||||||
def list_attribute_definitions(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
|
||||||
limit, offset = parse_page(limit, offset)
|
|
||||||
items, total = CatalogRepository(session).list_definitions(limit, offset)
|
|
||||||
return page([jsonable(item.__dict__) for item in items], limit, offset, total)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/sources")
|
|
||||||
def list_sources(session: Session = Depends(get_session)):
|
|
||||||
items = CatalogRepository(session).list_sources()
|
|
||||||
return {
|
|
||||||
"items": [{"id": item.id, "name": item.name, "adapter_type": item.adapter_type} for item in items],
|
|
||||||
"limit": len(items),
|
|
||||||
"offset": 0,
|
|
||||||
"total": len(items),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/scoring-models/{version}")
|
|
||||||
def get_scoring_model(version: str, session: Session = Depends(get_session)):
|
|
||||||
model = CatalogRepository(session).get_scoring_model(version)
|
|
||||||
if model is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "scoring model not found", {"version": version})
|
|
||||||
return {
|
|
||||||
"version": model.version,
|
|
||||||
"algorithm_id": model.algorithm_id,
|
|
||||||
"definition": model.definition,
|
|
||||||
"definition_hash": model.definition_hash,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/profiles", status_code=201)
|
|
||||||
def create_profile(payload: dict[str, Any] = Body(...), session: Session = Depends(get_session)):
|
|
||||||
if not isinstance(payload, dict):
|
|
||||||
raise AppError(422, "PROFILE_INVALID", "profile must be an object")
|
|
||||||
row = IngestionService(session).create_profile(payload)
|
|
||||||
return {
|
|
||||||
"id": row.id,
|
|
||||||
"version": row.version,
|
|
||||||
"name": row.name,
|
|
||||||
"category": row.category,
|
|
||||||
"scoring_version": row.scoring_version,
|
|
||||||
"definition": row.definition,
|
|
||||||
"definition_hash": row.definition_hash,
|
|
||||||
"created_at": row.created_at,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/profiles")
|
|
||||||
def list_profiles(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
|
||||||
limit, offset = parse_page(limit, offset)
|
|
||||||
items, total = CatalogRepository(session).list_profiles(limit, offset)
|
|
||||||
return page(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"id": row.id,
|
|
||||||
"version": row.version,
|
|
||||||
"name": row.name,
|
|
||||||
"category": row.category,
|
|
||||||
"scoring_version": row.scoring_version,
|
|
||||||
"definition_hash": row.definition_hash,
|
|
||||||
"created_at": row.created_at,
|
|
||||||
}
|
|
||||||
for row in items
|
|
||||||
],
|
|
||||||
limit,
|
|
||||||
offset,
|
|
||||||
total,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/profiles/{profile_id}")
|
|
||||||
def get_profile(profile_id: str, version: int | None = None, session: Session = Depends(get_session)):
|
|
||||||
if version is None:
|
|
||||||
raise AppError(422, "VERSION_REQUIRED", "query parameter version is required")
|
|
||||||
profile = CatalogRepository(session).get_profile(profile_id, version)
|
|
||||||
if profile is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "profile version not found")
|
|
||||||
return profile.definition
|
|
||||||
|
|
@ -1,125 +0,0 @@
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.api.schemas import (
|
|
||||||
BuyingProjectIn,
|
|
||||||
BuyingProjectPatch,
|
|
||||||
DiscoverySearchIn,
|
|
||||||
ProjectCriteriaIn,
|
|
||||||
page,
|
|
||||||
parse_page,
|
|
||||||
)
|
|
||||||
from app.persistence.db import get_session
|
|
||||||
from app.services.catalog_service import CatalogService
|
|
||||||
from app.services.discovery_service import CandidateRankingService, DiscoverySearchService
|
|
||||||
from app.services.project_service import ProjectService
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1")
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/product-categories")
|
|
||||||
def list_product_categories(session: Session = Depends(get_session)):
|
|
||||||
return {"items": CatalogService(session).list_product_categories()}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/product-categories/{product_category_id}/sources")
|
|
||||||
def list_category_sources(product_category_id: str, session: Session = Depends(get_session)):
|
|
||||||
return {"items": CatalogService(session).sources_for_category(product_category_id)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/product-categories/{product_category_id}/available-criteria")
|
|
||||||
def available_criteria_for_category(
|
|
||||||
product_category_id: str,
|
|
||||||
source: str = Query(...),
|
|
||||||
session: Session = Depends(get_session),
|
|
||||||
):
|
|
||||||
return CatalogService(session).available_criteria(product_category_id, source)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-templates")
|
|
||||||
def list_buying_templates(session: Session = Depends(get_session)):
|
|
||||||
return {"items": CatalogService(session).list_templates()}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/buying-projects", status_code=201)
|
|
||||||
def create_buying_project(payload: BuyingProjectIn, session: Session = Depends(get_session)):
|
|
||||||
return ProjectService(session).create(payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects")
|
|
||||||
def list_buying_projects(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
|
||||||
limit, offset = parse_page(limit, offset)
|
|
||||||
items, total = ProjectService(session).list(limit, offset)
|
|
||||||
return page(items, limit, offset, total)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}")
|
|
||||||
def get_buying_project(project_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return ProjectService(session).get(project_id)
|
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/buying-projects/{project_id}")
|
|
||||||
def patch_buying_project(project_id: UUID, payload: BuyingProjectPatch, session: Session = Depends(get_session)):
|
|
||||||
return ProjectService(session).patch(project_id, payload.model_dump(exclude_unset=True))
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/buying-projects/{project_id}/criteria", status_code=201)
|
|
||||||
def add_buying_project_criteria(
|
|
||||||
project_id: UUID,
|
|
||||||
payload: ProjectCriteriaIn,
|
|
||||||
session: Session = Depends(get_session),
|
|
||||||
):
|
|
||||||
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/buying-projects/{project_id}/criteria")
|
|
||||||
def put_buying_project_criteria(
|
|
||||||
project_id: UUID,
|
|
||||||
payload: ProjectCriteriaIn,
|
|
||||||
session: Session = Depends(get_session),
|
|
||||||
):
|
|
||||||
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}/criteria")
|
|
||||||
def get_buying_project_criteria(project_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return ProjectService(session).get_criteria(project_id)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}/available-criteria")
|
|
||||||
def available_criteria(project_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return ProjectService(session).available_criteria(project_id)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}/discovery-searches")
|
|
||||||
def list_project_searches(project_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return {"items": DiscoverySearchService(session).list_for_project(project_id)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/buying-projects/{project_id}/discovery-searches", status_code=201)
|
|
||||||
def create_project_search(project_id: UUID, payload: DiscoverySearchIn, session: Session = Depends(get_session)):
|
|
||||||
body = payload.model_dump()
|
|
||||||
body["buying_project_id"] = project_id
|
|
||||||
return DiscoverySearchService(session).create(body)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}/candidates")
|
|
||||||
def list_candidates(project_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return {"items": CandidateRankingService(session).list_candidates(project_id)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}")
|
|
||||||
def get_candidate(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return CandidateRankingService(session).get_candidate(project_id, candidate_id)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}/observations")
|
|
||||||
def list_candidate_observations(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return {"items": CandidateRankingService(session).list_observations(project_id, candidate_id)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/buying-projects/{project_id}/shortlist")
|
|
||||||
def list_shortlist(project_id: UUID, session: Session = Depends(get_session)):
|
|
||||||
return {"items": CandidateRankingService(session).list_candidates(project_id, shortlisted_only=True)}
|
|
||||||
|
|
@ -1,281 +0,0 @@
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict, Field, field_serializer
|
|
||||||
|
|
||||||
|
|
||||||
class StrictModel(BaseModel):
|
|
||||||
model_config = ConfigDict(extra="forbid")
|
|
||||||
|
|
||||||
|
|
||||||
class AttributeDefinitionIn(StrictModel):
|
|
||||||
key: str
|
|
||||||
value_type: str
|
|
||||||
canonical_unit: str | None = None
|
|
||||||
minimum: Decimal | None = None
|
|
||||||
maximum: Decimal | None = None
|
|
||||||
description: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class AttributeDefinitionOut(StrictModel):
|
|
||||||
key: str
|
|
||||||
value_type: str
|
|
||||||
canonical_unit: str | None
|
|
||||||
minimum: Decimal | None
|
|
||||||
maximum: Decimal | None
|
|
||||||
description: str | None
|
|
||||||
|
|
||||||
|
|
||||||
class ProductIn(StrictModel):
|
|
||||||
id: UUID | None = None
|
|
||||||
category: str
|
|
||||||
brand: str
|
|
||||||
name: str
|
|
||||||
model: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class ProductOut(StrictModel):
|
|
||||||
id: UUID
|
|
||||||
category: str
|
|
||||||
brand: str
|
|
||||||
name: str
|
|
||||||
model: str | None
|
|
||||||
created_at: datetime
|
|
||||||
|
|
||||||
|
|
||||||
class VariantIn(StrictModel):
|
|
||||||
id: UUID | None = None
|
|
||||||
name: str
|
|
||||||
manufacturer_sku: str | None = None
|
|
||||||
identifiers: dict[str, Any] = Field(default_factory=dict)
|
|
||||||
|
|
||||||
|
|
||||||
class AttributeIn(StrictModel):
|
|
||||||
id: UUID | None = None
|
|
||||||
attribute: str
|
|
||||||
raw_value: Any
|
|
||||||
normalized_value: Any | None = None
|
|
||||||
unit: str | None = None
|
|
||||||
normalization_status: str
|
|
||||||
normalizer_version: str | None = None
|
|
||||||
confidence: Decimal
|
|
||||||
source_id: str
|
|
||||||
source_reference: str
|
|
||||||
observed_at: datetime
|
|
||||||
normalization_error: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class AttributeOut(StrictModel):
|
|
||||||
id: UUID
|
|
||||||
variant_id: UUID
|
|
||||||
attribute: str
|
|
||||||
raw_value: Any
|
|
||||||
normalized_value: Any | None
|
|
||||||
unit: str | None
|
|
||||||
normalization_status: str
|
|
||||||
normalizer_version: str
|
|
||||||
confidence: Decimal
|
|
||||||
source_id: str
|
|
||||||
source_reference: str
|
|
||||||
observed_at: datetime
|
|
||||||
ingested_at: datetime
|
|
||||||
normalization_error: str | None
|
|
||||||
|
|
||||||
|
|
||||||
class VariantOut(StrictModel):
|
|
||||||
id: UUID
|
|
||||||
product_id: UUID
|
|
||||||
name: str
|
|
||||||
manufacturer_sku: str | None
|
|
||||||
identifiers: dict[str, Any]
|
|
||||||
created_at: datetime
|
|
||||||
attributes: list[AttributeOut]
|
|
||||||
|
|
||||||
|
|
||||||
class OfferIn(StrictModel):
|
|
||||||
id: UUID | None = None
|
|
||||||
source_id: str
|
|
||||||
external_id: str
|
|
||||||
seller: str
|
|
||||||
url: str | None = None
|
|
||||||
market: str
|
|
||||||
currency: str
|
|
||||||
condition: str
|
|
||||||
|
|
||||||
|
|
||||||
class OfferOut(StrictModel):
|
|
||||||
id: UUID
|
|
||||||
variant_id: UUID
|
|
||||||
source_id: str
|
|
||||||
external_id: str
|
|
||||||
seller: str
|
|
||||||
url: str | None
|
|
||||||
market: str
|
|
||||||
currency: str
|
|
||||||
condition: str
|
|
||||||
created_at: datetime
|
|
||||||
|
|
||||||
|
|
||||||
class PriceIn(StrictModel):
|
|
||||||
id: UUID | None = None
|
|
||||||
source_id: str
|
|
||||||
external_observation_id: str
|
|
||||||
observed_at: datetime
|
|
||||||
item_price: Decimal
|
|
||||||
shipping_price: Decimal | None = None
|
|
||||||
currency: str
|
|
||||||
availability: str
|
|
||||||
raw_payload: Any
|
|
||||||
confidence: Decimal
|
|
||||||
|
|
||||||
@field_serializer("item_price", "shipping_price")
|
|
||||||
def serialize_money(self, value: Decimal | None) -> str | None:
|
|
||||||
return None if value is None else format(value, "f")
|
|
||||||
|
|
||||||
|
|
||||||
class PriceOut(StrictModel):
|
|
||||||
id: UUID
|
|
||||||
offer_id: UUID
|
|
||||||
source_id: str
|
|
||||||
external_observation_id: str
|
|
||||||
observed_at: datetime
|
|
||||||
ingested_at: datetime
|
|
||||||
item_price: Decimal
|
|
||||||
shipping_price: Decimal | None
|
|
||||||
currency: str
|
|
||||||
availability: str
|
|
||||||
raw_payload: Any
|
|
||||||
confidence: Decimal
|
|
||||||
|
|
||||||
@field_serializer("item_price", "shipping_price")
|
|
||||||
def serialize_money(self, value: Decimal | None) -> str | None:
|
|
||||||
return None if value is None else format(Decimal(value).quantize(Decimal("0.01")), "f")
|
|
||||||
|
|
||||||
|
|
||||||
class EvaluateIn(StrictModel):
|
|
||||||
profile_id: str
|
|
||||||
profile_version: int
|
|
||||||
offer_id: UUID
|
|
||||||
as_of: datetime | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class BuyingProjectIn(StrictModel):
|
|
||||||
id: UUID | None = None
|
|
||||||
name: str
|
|
||||||
product_category_id: str | None = None
|
|
||||||
market: str | None = None
|
|
||||||
currency: str | None = None
|
|
||||||
target_budget: Decimal | None = None
|
|
||||||
template_id: str | None = None
|
|
||||||
template_version: int | None = None
|
|
||||||
profile_id: str | None = None
|
|
||||||
profile_version: int | None = None
|
|
||||||
workspace_id: str | None = None
|
|
||||||
shortlist_size: int = 5
|
|
||||||
schedule_interval_minutes: int | None = None
|
|
||||||
criteria: dict[str, Any]
|
|
||||||
|
|
||||||
|
|
||||||
class BuyingProjectPatch(StrictModel):
|
|
||||||
name: str | None = None
|
|
||||||
target_budget: Decimal | None = None
|
|
||||||
shortlist_size: int | None = None
|
|
||||||
status: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class ProjectCriteriaIn(StrictModel):
|
|
||||||
items: list[dict[str, Any]]
|
|
||||||
|
|
||||||
|
|
||||||
class SourceCategoryIn(StrictModel):
|
|
||||||
source_url: str
|
|
||||||
name: str | None = None
|
|
||||||
product_category_id: str | None = None
|
|
||||||
product_category_name: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class CriterionDefinitionIn(StrictModel):
|
|
||||||
key: str
|
|
||||||
value_type: str
|
|
||||||
unit: str | None = None
|
|
||||||
semantic: str | None = None
|
|
||||||
category: str | None = None
|
|
||||||
description: str | None = None
|
|
||||||
origin: str | None = None
|
|
||||||
source: str | None = None
|
|
||||||
product_category_id: str | None = None
|
|
||||||
label: str | None = None
|
|
||||||
scoring_attribute_key: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class DiscoverySearchIn(StrictModel):
|
|
||||||
buying_project_id: UUID | None = None
|
|
||||||
source_id: str
|
|
||||||
source_category_id: UUID
|
|
||||||
schedule_interval_minutes: int = 180
|
|
||||||
broad_result_threshold: int = 300
|
|
||||||
|
|
||||||
|
|
||||||
class DiscoverySearchPatch(StrictModel):
|
|
||||||
schedule_interval_minutes: int | None = None
|
|
||||||
status: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class SourceAttributeMappingIn(StrictModel):
|
|
||||||
source_id: str
|
|
||||||
source_category_id: UUID | None = None
|
|
||||||
raw_name: str
|
|
||||||
criterion_definition_id: UUID
|
|
||||||
status: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class SourceAttributeMappingPatch(StrictModel):
|
|
||||||
raw_name: str | None = None
|
|
||||||
criterion_definition_id: UUID | None = None
|
|
||||||
status: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class CriterionMappingIn(StrictModel):
|
|
||||||
criterion_definition_id: UUID | None = None
|
|
||||||
criterion_key: str | None = None
|
|
||||||
source_id: str
|
|
||||||
source_category_id: UUID
|
|
||||||
external_filter_key: str
|
|
||||||
source_value: str | None = None
|
|
||||||
encoding: dict[str, Any] = Field(default_factory=lambda: {"url_param": "xf"})
|
|
||||||
status: str | None = None
|
|
||||||
value_type: str | None = None
|
|
||||||
unit: str | None = None
|
|
||||||
semantic: str | None = None
|
|
||||||
description: str | None = None
|
|
||||||
operator: str | None = None
|
|
||||||
probe_value: Any = None
|
|
||||||
|
|
||||||
|
|
||||||
class CriterionMappingPatch(StrictModel):
|
|
||||||
source_value: str | None = None
|
|
||||||
encoding: dict[str, Any] | None = None
|
|
||||||
status: str | None = None
|
|
||||||
probe_value: Any = None
|
|
||||||
operator: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class Page(StrictModel):
|
|
||||||
items: list[Any]
|
|
||||||
limit: int
|
|
||||||
offset: int
|
|
||||||
total: int
|
|
||||||
|
|
||||||
|
|
||||||
def page(items: list[Any], limit: int, offset: int, total: int) -> dict[str, Any]:
|
|
||||||
return {"items": items, "limit": limit, "offset": offset, "total": total}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_page(limit: int = 50, offset: int = 0) -> tuple[int, int]:
|
|
||||||
from app.errors import AppError
|
|
||||||
|
|
||||||
if limit < 1 or limit > 200 or offset < 0:
|
|
||||||
raise AppError(422, "PAGINATION", "limit must be 1-200 and offset >= 0")
|
|
||||||
return limit, offset
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
from app.settings import get_settings
|
|
||||||
|
|
||||||
|
|
||||||
def now_utc() -> datetime:
|
|
||||||
override = get_settings().clock_now
|
|
||||||
if override is not None:
|
|
||||||
if override.tzinfo is None:
|
|
||||||
return override.replace(tzinfo=timezone.utc)
|
|
||||||
return override.astimezone(timezone.utc)
|
|
||||||
return datetime.now(timezone.utc)
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
from app.discovery.criteria import parse_project_criteria
|
|
||||||
from app.discovery.diff import diff_run, next_lifecycle, observations_changed
|
|
||||||
from app.discovery.enums import (
|
|
||||||
CandidateStatus,
|
|
||||||
DiffStatus,
|
|
||||||
LifecycleStatus,
|
|
||||||
MappingStatus,
|
|
||||||
MatchStatus,
|
|
||||||
ProjectStatus,
|
|
||||||
RunStatus,
|
|
||||||
SearchStatus,
|
|
||||||
)
|
|
||||||
from app.discovery.identity import IdentityDecision, IdentitySignals, decide_identity, spec_fingerprint
|
|
||||||
from app.discovery.mapping import verify_roundtrip
|
|
||||||
from app.discovery.ranking import rank_candidates
|
|
||||||
from app.discovery.urlgen import generate_search_url
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
"CandidateStatus",
|
|
||||||
"DiffStatus",
|
|
||||||
"IdentityDecision",
|
|
||||||
"IdentitySignals",
|
|
||||||
"LifecycleStatus",
|
|
||||||
"MappingStatus",
|
|
||||||
"MatchStatus",
|
|
||||||
"ProjectStatus",
|
|
||||||
"RunStatus",
|
|
||||||
"SearchStatus",
|
|
||||||
"decide_identity",
|
|
||||||
"diff_run",
|
|
||||||
"generate_search_url",
|
|
||||||
"next_lifecycle",
|
|
||||||
"observations_changed",
|
|
||||||
"parse_project_criteria",
|
|
||||||
"rank_candidates",
|
|
||||||
"spec_fingerprint",
|
|
||||||
"verify_roundtrip",
|
|
||||||
]
|
|
||||||
|
|
@ -1,51 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
SPEC_ALIASES = {
|
|
||||||
"ram_gb": ("ram", "arbeitsspeicher"),
|
|
||||||
"storage_gb": ("speicher", "ssd", "festplatte"),
|
|
||||||
"weight_kg": ("gewicht",),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def merge_observation_attributes(
|
|
||||||
raw_attributes: dict[str, Any],
|
|
||||||
normalized_attributes: dict[str, Any],
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
items: list[dict[str, Any]] = []
|
|
||||||
used_norm: set[str] = set()
|
|
||||||
for name, raw_value in raw_attributes.items():
|
|
||||||
if raw_value is None or str(raw_value).strip() == "":
|
|
||||||
continue
|
|
||||||
fold = str(name).casefold().rstrip(":")
|
|
||||||
norm_key = next((key for key, aliases in SPEC_ALIASES.items() if fold in aliases), None)
|
|
||||||
norm_value = normalized_attributes.get(norm_key) if norm_key else None
|
|
||||||
if norm_key:
|
|
||||||
used_norm.add(norm_key)
|
|
||||||
items.append(
|
|
||||||
{
|
|
||||||
"attribute_key": norm_key or fold.replace(" ", "_"),
|
|
||||||
"raw_name": str(name).rstrip(":"),
|
|
||||||
"raw_value": raw_value,
|
|
||||||
"normalized_value": norm_value,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
for key, value in normalized_attributes.items():
|
|
||||||
if key in used_norm or value is None:
|
|
||||||
continue
|
|
||||||
items.append(
|
|
||||||
{
|
|
||||||
"attribute_key": key,
|
|
||||||
"raw_name": key,
|
|
||||||
"raw_value": None,
|
|
||||||
"normalized_value": value,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return items
|
|
||||||
|
|
||||||
|
|
||||||
def is_derived_duplicate(name: str, present: set[str]) -> bool:
|
|
||||||
fold = name.casefold()
|
|
||||||
aliases = SPEC_ALIASES.get(fold)
|
|
||||||
return bool(aliases and present.intersection(aliases))
|
|
||||||
|
|
@ -1,77 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from app.domain.validation import DomainValidationError, canonical_json, sha256_text
|
|
||||||
|
|
||||||
|
|
||||||
def parse_project_criteria(definition: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
items = definition.get("items")
|
|
||||||
if not isinstance(items, list) or not items:
|
|
||||||
raise DomainValidationError("criteria.items must be a non-empty list")
|
|
||||||
seen: set[str] = set()
|
|
||||||
canonical_items: list[dict[str, Any]] = []
|
|
||||||
for raw in items:
|
|
||||||
criterion_id = raw.get("criterion_definition_id")
|
|
||||||
key = raw.get("criterion")
|
|
||||||
item_id = str(raw.get("id") or criterion_id or key or "")
|
|
||||||
if not item_id:
|
|
||||||
raise DomainValidationError("each criterion needs id, criterion_definition_id or criterion")
|
|
||||||
if item_id in seen:
|
|
||||||
raise DomainValidationError("duplicate criterion id", {"id": item_id})
|
|
||||||
seen.add(item_id)
|
|
||||||
if not criterion_id and not key:
|
|
||||||
raise DomainValidationError("criterion_definition_id or criterion is required", {"id": item_id})
|
|
||||||
if not raw.get("operator"):
|
|
||||||
raise DomainValidationError("operator is required", {"id": item_id})
|
|
||||||
enabled = raw.get("enabled_for_discovery")
|
|
||||||
if enabled is None:
|
|
||||||
enabled = raw.get("mandatory", True)
|
|
||||||
canonical_items.append(
|
|
||||||
{
|
|
||||||
"id": item_id,
|
|
||||||
"criterion_definition_id": None if criterion_id is None else UUID(str(criterion_id)),
|
|
||||||
"criterion": None if key is None else str(key),
|
|
||||||
"operator": str(raw["operator"]),
|
|
||||||
"value": raw.get("value"),
|
|
||||||
"enabled_for_discovery": bool(enabled),
|
|
||||||
"mandatory": bool(enabled),
|
|
||||||
"value_type": raw.get("value_type"),
|
|
||||||
"unit": raw.get("unit"),
|
|
||||||
"semantic": raw.get("semantic"),
|
|
||||||
"category": raw.get("category"),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
canonical = {
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"id": item["id"],
|
|
||||||
"criterion_definition_id": None
|
|
||||||
if item["criterion_definition_id"] is None
|
|
||||||
else str(item["criterion_definition_id"]),
|
|
||||||
"criterion": item["criterion"],
|
|
||||||
"operator": item["operator"],
|
|
||||||
"value": item["value"],
|
|
||||||
"enabled_for_discovery": item["enabled_for_discovery"],
|
|
||||||
}
|
|
||||||
for item in canonical_items
|
|
||||||
]
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
"items": canonical_items,
|
|
||||||
"definition": canonical,
|
|
||||||
"definition_hash": sha256_text(canonical_json(canonical)),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def criterion_payload(row) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": row.item_id,
|
|
||||||
"criterion_definition_id": row.criterion_definition_id,
|
|
||||||
"criterion": row.criterion_key,
|
|
||||||
"operator": row.operator,
|
|
||||||
"value": row.value,
|
|
||||||
"enabled_for_discovery": row.enabled_for_discovery,
|
|
||||||
"mandatory": row.enabled_for_discovery,
|
|
||||||
}
|
|
||||||
|
|
@ -1,51 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID, uuid4
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.persistence.discovery_models import CriterionDefinitionRecord
|
|
||||||
|
|
||||||
|
|
||||||
def get_definition_by_key(session: Session, key: str) -> CriterionDefinitionRecord | None:
|
|
||||||
return session.scalars(select(CriterionDefinitionRecord).where(CriterionDefinitionRecord.key == key)).first()
|
|
||||||
|
|
||||||
|
|
||||||
def get_definition(session: Session, definition_id: UUID) -> CriterionDefinitionRecord | None:
|
|
||||||
return session.get(CriterionDefinitionRecord, definition_id)
|
|
||||||
|
|
||||||
|
|
||||||
def upsert_definition(session: Session, payload: dict[str, Any]) -> CriterionDefinitionRecord:
|
|
||||||
key = str(payload["key"])
|
|
||||||
existing = get_definition_by_key(session, key)
|
|
||||||
if existing is not None:
|
|
||||||
if payload.get("label"):
|
|
||||||
existing.label = str(payload["label"])
|
|
||||||
if payload.get("scoring_attribute_key"):
|
|
||||||
existing.scoring_attribute_key = str(payload["scoring_attribute_key"])
|
|
||||||
if payload.get("product_category_id"):
|
|
||||||
existing.product_category_id = str(payload["product_category_id"])
|
|
||||||
if payload.get("origin"):
|
|
||||||
existing.origin = str(payload["origin"])
|
|
||||||
if payload.get("source"):
|
|
||||||
existing.source = str(payload["source"])
|
|
||||||
return existing
|
|
||||||
row = CriterionDefinitionRecord(
|
|
||||||
id=payload.get("id") or uuid4(),
|
|
||||||
key=key,
|
|
||||||
origin=str(payload.get("origin") or "SOURCE"),
|
|
||||||
source=payload.get("source"),
|
|
||||||
product_category_id=payload.get("product_category_id"),
|
|
||||||
label=str(payload.get("label") or key),
|
|
||||||
value_type=str(payload["value_type"]),
|
|
||||||
unit=payload.get("unit"),
|
|
||||||
semantic=payload.get("semantic"),
|
|
||||||
category=payload.get("category") or payload.get("product_category_id"),
|
|
||||||
description=payload.get("description"),
|
|
||||||
scoring_attribute_key=payload.get("scoring_attribute_key"),
|
|
||||||
)
|
|
||||||
session.add(row)
|
|
||||||
session.flush()
|
|
||||||
return row
|
|
||||||
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from app.discovery.enums import DiffStatus, LifecycleStatus, RunStatus
|
|
||||||
|
|
||||||
|
|
||||||
RELEVANT_ATTR_IGNORE = {"title", "html", "presentation", "raw_title"}
|
|
||||||
|
|
||||||
|
|
||||||
def observations_changed(previous: dict[str, Any] | None, current: dict[str, Any] | None) -> bool:
|
|
||||||
if previous is None or current is None:
|
|
||||||
return previous != current
|
|
||||||
if _money(previous.get("best_price")) != _money(current.get("best_price")):
|
|
||||||
return True
|
|
||||||
if previous.get("availability") != current.get("availability"):
|
|
||||||
return True
|
|
||||||
if previous.get("seller_count") != current.get("seller_count"):
|
|
||||||
return True
|
|
||||||
prev_attrs = _technical(previous.get("normalized_attributes") or {})
|
|
||||||
curr_attrs = _technical(current.get("normalized_attributes") or {})
|
|
||||||
return prev_attrs != curr_attrs
|
|
||||||
|
|
||||||
|
|
||||||
def _money(value: Any) -> Decimal | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
return Decimal(str(value))
|
|
||||||
|
|
||||||
|
|
||||||
def _technical(attrs: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
return {key: attrs[key] for key in sorted(attrs) if key not in RELEVANT_ATTR_IGNORE}
|
|
||||||
|
|
||||||
|
|
||||||
def diff_run(
|
|
||||||
*,
|
|
||||||
previous_ids: set[Any],
|
|
||||||
current_ids: set[Any],
|
|
||||||
previous_obs: dict[Any, dict[str, Any]],
|
|
||||||
current_obs: dict[Any, dict[str, Any]],
|
|
||||||
run_status: str,
|
|
||||||
) -> dict[Any, DiffStatus]:
|
|
||||||
result: dict[Any, DiffStatus] = {}
|
|
||||||
for ref_id in current_ids:
|
|
||||||
if ref_id not in previous_ids:
|
|
||||||
result[ref_id] = DiffStatus.NEW
|
|
||||||
elif observations_changed(previous_obs.get(ref_id), current_obs.get(ref_id)):
|
|
||||||
result[ref_id] = DiffStatus.CHANGED
|
|
||||||
else:
|
|
||||||
result[ref_id] = DiffStatus.UNCHANGED
|
|
||||||
if run_status == RunStatus.SUCCESS.value:
|
|
||||||
for ref_id in previous_ids - current_ids:
|
|
||||||
result[ref_id] = DiffStatus.MISSING
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def next_lifecycle(
|
|
||||||
*,
|
|
||||||
current: str,
|
|
||||||
missing_in_success_run: bool,
|
|
||||||
consecutive_missing: int,
|
|
||||||
reappeared: bool,
|
|
||||||
) -> tuple[str, int]:
|
|
||||||
if current == LifecycleStatus.ARCHIVED.value:
|
|
||||||
return current, consecutive_missing
|
|
||||||
if reappeared:
|
|
||||||
return LifecycleStatus.ACTIVE.value, 0
|
|
||||||
if missing_in_success_run:
|
|
||||||
streak = consecutive_missing + 1
|
|
||||||
if streak >= 3:
|
|
||||||
return LifecycleStatus.DISAPPEARED.value, streak
|
|
||||||
return LifecycleStatus.ACTIVE.value, streak
|
|
||||||
return current, consecutive_missing
|
|
||||||
|
|
@ -1,87 +0,0 @@
|
||||||
from enum import StrEnum
|
|
||||||
|
|
||||||
|
|
||||||
class ProjectStatus(StrEnum):
|
|
||||||
ACTIVE = "ACTIVE"
|
|
||||||
PURCHASED = "PURCHASED"
|
|
||||||
ARCHIVED = "ARCHIVED"
|
|
||||||
|
|
||||||
|
|
||||||
class TaxonomyStatus(StrEnum):
|
|
||||||
NOT_LEARNED = "NOT_LEARNED"
|
|
||||||
LEARNING = "LEARNING"
|
|
||||||
READY = "READY"
|
|
||||||
DEGRADED = "DEGRADED"
|
|
||||||
BROKEN = "BROKEN"
|
|
||||||
BLOCKED = "BLOCKED"
|
|
||||||
|
|
||||||
|
|
||||||
class MappingStatus(StrEnum):
|
|
||||||
DISCOVERED = "DISCOVERED"
|
|
||||||
LEARNED_UNVERIFIED = "LEARNED_UNVERIFIED"
|
|
||||||
VERIFIED = "VERIFIED"
|
|
||||||
BROKEN = "BROKEN"
|
|
||||||
DISABLED = "DISABLED"
|
|
||||||
|
|
||||||
|
|
||||||
class FilterType(StrEnum):
|
|
||||||
BOOLEAN = "BOOLEAN"
|
|
||||||
ENUM = "ENUM"
|
|
||||||
MULTI_ENUM = "MULTI_ENUM"
|
|
||||||
NUMERIC_RANGE = "NUMERIC_RANGE"
|
|
||||||
NUMERIC_THRESHOLD = "NUMERIC_THRESHOLD"
|
|
||||||
TEXT = "TEXT"
|
|
||||||
UNKNOWN = "UNKNOWN"
|
|
||||||
|
|
||||||
|
|
||||||
class CriterionOrigin(StrEnum):
|
|
||||||
SOURCE = "SOURCE"
|
|
||||||
INTERNAL = "INTERNAL"
|
|
||||||
|
|
||||||
|
|
||||||
class SearchStatus(StrEnum):
|
|
||||||
ACTIVE = "ACTIVE"
|
|
||||||
OUT_OF_SYNC = "OUT_OF_SYNC"
|
|
||||||
BLOCKED = "BLOCKED"
|
|
||||||
DISABLED = "DISABLED"
|
|
||||||
TOO_BROAD = "TOO_BROAD"
|
|
||||||
PAUSED = "PAUSED"
|
|
||||||
|
|
||||||
|
|
||||||
class RunStatus(StrEnum):
|
|
||||||
RUNNING = "RUNNING"
|
|
||||||
SUCCESS = "SUCCESS"
|
|
||||||
PARTIAL = "PARTIAL"
|
|
||||||
FAILED = "FAILED"
|
|
||||||
BLOCKED = "BLOCKED"
|
|
||||||
TOO_BROAD = "TOO_BROAD"
|
|
||||||
|
|
||||||
|
|
||||||
class RankingStatus(StrEnum):
|
|
||||||
NOT_CONFIGURED = "NOT_CONFIGURED"
|
|
||||||
CONFIGURED = "CONFIGURED"
|
|
||||||
|
|
||||||
|
|
||||||
class DiffStatus(StrEnum):
|
|
||||||
NEW = "NEW"
|
|
||||||
CHANGED = "CHANGED"
|
|
||||||
UNCHANGED = "UNCHANGED"
|
|
||||||
MISSING = "MISSING"
|
|
||||||
|
|
||||||
|
|
||||||
class LifecycleStatus(StrEnum):
|
|
||||||
ACTIVE = "ACTIVE"
|
|
||||||
DISAPPEARED = "DISAPPEARED"
|
|
||||||
ARCHIVED = "ARCHIVED"
|
|
||||||
|
|
||||||
|
|
||||||
class MatchStatus(StrEnum):
|
|
||||||
MATCHED = "MATCHED"
|
|
||||||
MATCH_UNCERTAIN = "MATCH_UNCERTAIN"
|
|
||||||
UNCERTAIN = "MATCH_UNCERTAIN"
|
|
||||||
NEW_REFERENCE = "NEW_REFERENCE"
|
|
||||||
|
|
||||||
|
|
||||||
class CandidateStatus(StrEnum):
|
|
||||||
CANDIDATE = "CANDIDATE"
|
|
||||||
SHORTLISTED = "SHORTLISTED"
|
|
||||||
|
|
@ -1,81 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from app.discovery.enums import MatchStatus
|
|
||||||
|
|
||||||
PRESENTATION_KEYS = {"title", "html", "presentation", "raw_title"}
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_title(title: str) -> str:
|
|
||||||
return re.sub(r"\s+", " ", title.casefold().strip())
|
|
||||||
|
|
||||||
|
|
||||||
def spec_fingerprint(normalized_attributes: dict[str, Any], keys: tuple[str, ...] | None = None) -> str:
|
|
||||||
"""Fingerprint from whatever technical keys the observation actually has."""
|
|
||||||
use = keys if keys is not None else tuple(sorted(key for key in normalized_attributes if key not in PRESENTATION_KEYS))
|
|
||||||
parts = []
|
|
||||||
for key in use:
|
|
||||||
value = normalized_attributes.get(key)
|
|
||||||
parts.append(f"{key}={'' if value is None else str(value).casefold().strip()}")
|
|
||||||
return "|".join(parts)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class IdentitySignals:
|
|
||||||
source_external_id: str | None
|
|
||||||
gtin_ean: str | None
|
|
||||||
manufacturer_sku: str | None
|
|
||||||
source_url: str | None
|
|
||||||
title: str
|
|
||||||
manufacturer: str | None
|
|
||||||
model: str | None
|
|
||||||
fingerprint: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class IdentityDecision:
|
|
||||||
match_status: MatchStatus
|
|
||||||
existing_id: Any | None
|
|
||||||
reason: str
|
|
||||||
|
|
||||||
|
|
||||||
def decide_identity(incoming: IdentitySignals, known: list[tuple[Any, IdentitySignals]]) -> IdentityDecision:
|
|
||||||
"""Conservative matching. Similar titles never merge by themselves."""
|
|
||||||
for ref_id, existing in known:
|
|
||||||
if incoming.source_external_id and incoming.source_external_id == existing.source_external_id:
|
|
||||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_external_id")
|
|
||||||
if incoming.gtin_ean and incoming.gtin_ean == existing.gtin_ean:
|
|
||||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "gtin_ean")
|
|
||||||
if incoming.manufacturer_sku and incoming.manufacturer_sku == existing.manufacturer_sku:
|
|
||||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "manufacturer_sku")
|
|
||||||
if incoming.source_url and incoming.source_url == existing.source_url:
|
|
||||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_url")
|
|
||||||
|
|
||||||
title_hits = [
|
|
||||||
(ref_id, existing)
|
|
||||||
for ref_id, existing in known
|
|
||||||
if normalize_title(incoming.title) == normalize_title(existing.title)
|
|
||||||
]
|
|
||||||
if len(title_hits) == 1 and incoming.fingerprint and incoming.fingerprint == title_hits[0][1].fingerprint:
|
|
||||||
return IdentityDecision(MatchStatus.MATCHED, title_hits[0][0], "title+fingerprint")
|
|
||||||
|
|
||||||
manufacturer_model = [
|
|
||||||
(ref_id, existing)
|
|
||||||
for ref_id, existing in known
|
|
||||||
if incoming.manufacturer
|
|
||||||
and incoming.model
|
|
||||||
and incoming.manufacturer.casefold() == (existing.manufacturer or "").casefold()
|
|
||||||
and incoming.model.casefold() == (existing.model or "").casefold()
|
|
||||||
]
|
|
||||||
if len(manufacturer_model) == 1:
|
|
||||||
existing = manufacturer_model[0][1]
|
|
||||||
if incoming.fingerprint and existing.fingerprint and incoming.fingerprint != existing.fingerprint:
|
|
||||||
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "manufacturer+model conflict with specs")
|
|
||||||
return IdentityDecision(MatchStatus.MATCHED, manufacturer_model[0][0], "manufacturer+model")
|
|
||||||
|
|
||||||
if title_hits:
|
|
||||||
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "title only")
|
|
||||||
return IdentityDecision(MatchStatus.NEW_REFERENCE, None, "no match")
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import parse_qsl, urlsplit
|
|
||||||
|
|
||||||
from app.discovery.enums import MappingStatus
|
|
||||||
from app.discovery.urlgen import decode_xf_roundtrip, generate_search_url
|
|
||||||
|
|
||||||
|
|
||||||
def verify_roundtrip(
|
|
||||||
*,
|
|
||||||
category_url: str,
|
|
||||||
criterion: dict[str, Any],
|
|
||||||
mapping: dict[str, Any],
|
|
||||||
) -> bool:
|
|
||||||
"""True when encoding a single mapping round-trips to the same source parameter."""
|
|
||||||
probe = {**mapping, "status": MappingStatus.VERIFIED.value}
|
|
||||||
url, internal = generate_search_url(
|
|
||||||
category_url=category_url,
|
|
||||||
criteria_items=[criterion],
|
|
||||||
mappings=[probe],
|
|
||||||
)
|
|
||||||
if internal:
|
|
||||||
return False
|
|
||||||
encoding = mapping.get("encoding") or {}
|
|
||||||
param = str(encoding.get("url_param") or "xf")
|
|
||||||
if param == "xf":
|
|
||||||
expected_key = str(mapping["external_filter_key"])
|
|
||||||
expected_value = str(mapping.get("source_value") or "")
|
|
||||||
decoded = decode_xf_roundtrip(url)
|
|
||||||
return any(key == expected_key and value == expected_value.replace("+", " ") for key, value in decoded)
|
|
||||||
expected = str(criterion.get("value"))
|
|
||||||
query = parse_qsl(urlsplit(url).query, keep_blank_values=True)
|
|
||||||
return any(name == param and value == expected for name, value in query)
|
|
||||||
|
|
@ -1,100 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID, uuid4
|
|
||||||
|
|
||||||
from app.clock import now_utc
|
|
||||||
from app.domain.attributes import select_attribute_snapshot
|
|
||||||
from app.domain.constraints import evaluate_constraint
|
|
||||||
from app.domain.enums import Eligibility, NormalizationStatus
|
|
||||||
from app.domain.models import AttributeStatement, Product
|
|
||||||
from app.scoring.engine import _product_eligibility
|
|
||||||
from app.scoring.mathutil import round_score
|
|
||||||
from app.scoring.product import score_product
|
|
||||||
|
|
||||||
|
|
||||||
def rank_candidates(
|
|
||||||
*,
|
|
||||||
profile,
|
|
||||||
scoring_model,
|
|
||||||
definitions,
|
|
||||||
candidates: list[dict[str, Any]],
|
|
||||||
shortlist_size: int,
|
|
||||||
as_of: datetime | None = None,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
scored: list[dict[str, Any]] = []
|
|
||||||
when = as_of or now_utc()
|
|
||||||
for candidate in candidates:
|
|
||||||
attrs = _statements(candidate, when)
|
|
||||||
snapshot = select_attribute_snapshot(
|
|
||||||
attrs,
|
|
||||||
as_of=when,
|
|
||||||
snapshot_cutoff=when,
|
|
||||||
)
|
|
||||||
constraint_results = tuple(
|
|
||||||
evaluate_constraint(
|
|
||||||
constraint,
|
|
||||||
snapshot.get(constraint.attribute),
|
|
||||||
definitions[constraint.attribute],
|
|
||||||
scoring_model.minimum_confidence,
|
|
||||||
)
|
|
||||||
for constraint in profile.constraints
|
|
||||||
if constraint.attribute in definitions
|
|
||||||
)
|
|
||||||
product = Product(
|
|
||||||
id=uuid4(),
|
|
||||||
category=profile.category,
|
|
||||||
brand=str(candidate.get("manufacturer") or "unknown"),
|
|
||||||
name=str(candidate.get("source_title") or candidate.get("name") or "candidate"),
|
|
||||||
model=candidate.get("model"),
|
|
||||||
created_at=when,
|
|
||||||
)
|
|
||||||
_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
|
|
||||||
product_score, coverage, _components = score_product(
|
|
||||||
profile.preferences,
|
|
||||||
snapshot,
|
|
||||||
constraint_results,
|
|
||||||
scoring_model,
|
|
||||||
)
|
|
||||||
scored.append(
|
|
||||||
{
|
|
||||||
**candidate,
|
|
||||||
"product_eligibility": product_eligibility.value,
|
|
||||||
"product_score": round_score(product_score),
|
|
||||||
"product_coverage": coverage,
|
|
||||||
"eligible": product_eligibility == Eligibility.ELIGIBLE,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
scored.sort(key=lambda item: (not item["eligible"], -(item["product_score"] or Decimal("0"))))
|
|
||||||
for index, item in enumerate(scored):
|
|
||||||
item["candidate_status"] = "SHORTLISTED" if index < shortlist_size and item["eligible"] else "CANDIDATE"
|
|
||||||
item["rank"] = index + 1
|
|
||||||
return scored
|
|
||||||
|
|
||||||
|
|
||||||
def _statements(candidate: dict[str, Any], when: datetime) -> tuple[AttributeStatement, ...]:
|
|
||||||
raw = candidate.get("normalized_attributes") or {}
|
|
||||||
now = when
|
|
||||||
items = []
|
|
||||||
variant_id = candidate.get("variant_id") or uuid4()
|
|
||||||
for key, value in raw.items():
|
|
||||||
items.append(
|
|
||||||
AttributeStatement(
|
|
||||||
id=uuid4(),
|
|
||||||
variant_id=variant_id if isinstance(variant_id, UUID) else uuid4(),
|
|
||||||
attribute=key,
|
|
||||||
raw_value=value,
|
|
||||||
normalized_value=value,
|
|
||||||
unit=None,
|
|
||||||
normalization_status=NormalizationStatus.NORMALIZED,
|
|
||||||
normalizer_version="discovery-v1",
|
|
||||||
confidence=Decimal("1"),
|
|
||||||
source_id="geizhals",
|
|
||||||
source_reference="discovery",
|
|
||||||
observed_at=now,
|
|
||||||
ingested_at=now,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return tuple(items)
|
|
||||||
|
|
@ -1,171 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlsplit, urlunsplit
|
|
||||||
|
|
||||||
from app.discovery.enums import MappingStatus
|
|
||||||
|
|
||||||
|
|
||||||
def generate_search_url(
|
|
||||||
*,
|
|
||||||
category_url: str,
|
|
||||||
criteria_items: list[dict[str, Any]],
|
|
||||||
mappings: list[dict[str, Any]],
|
|
||||||
) -> tuple[str, list[str]]:
|
|
||||||
"""Build a source URL from VERIFIED mappings only. INTERNAL_ONLY keys are skipped."""
|
|
||||||
by_criterion = {item["criterion_key"]: item for item in mappings if item.get("status") == MappingStatus.VERIFIED.value}
|
|
||||||
fragments: list[tuple[str, str]] = []
|
|
||||||
internal_only: list[str] = []
|
|
||||||
for criterion in criteria_items:
|
|
||||||
key = str(criterion["criterion"])
|
|
||||||
mapping = by_criterion.get(key)
|
|
||||||
if mapping is None:
|
|
||||||
internal_only.append(key)
|
|
||||||
continue
|
|
||||||
fragments.extend(_encode_mapping(mapping, criterion))
|
|
||||||
fragments.sort()
|
|
||||||
base = urlsplit(category_url)
|
|
||||||
query = dict(parse_qsl(base.query, keep_blank_values=True))
|
|
||||||
extra: list[tuple[str, str]] = []
|
|
||||||
xf_values: list[str] = []
|
|
||||||
for name, value in fragments:
|
|
||||||
if name == "xf":
|
|
||||||
xf_values.append(value)
|
|
||||||
continue
|
|
||||||
if name in query:
|
|
||||||
query[name] = value
|
|
||||||
else:
|
|
||||||
extra.append((name, value))
|
|
||||||
if xf_values:
|
|
||||||
extra.append(("xf", "~".join(xf_values)))
|
|
||||||
encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote)
|
|
||||||
url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment))
|
|
||||||
return url, internal_only
|
|
||||||
|
|
||||||
|
|
||||||
def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
|
||||||
encoding = mapping.get("encoding") or {}
|
|
||||||
param = str(encoding.get("url_param") or "xf")
|
|
||||||
if encoding.get("encoding_type") == "RANGE_PARAMETERS" or param in {"bpmin", "bpmax"}:
|
|
||||||
return _encode_range(encoding, criterion)
|
|
||||||
if encoding.get("encoding_type") == "RANGE_TOKEN":
|
|
||||||
return _encode_range_token(mapping, criterion)
|
|
||||||
if param == "xf":
|
|
||||||
return _encode_xf(mapping, criterion)
|
|
||||||
if param in {"minp", "maxp", "prmin", "prmax", "asuch", "fs"}:
|
|
||||||
return [(param, str(criterion.get("value")))]
|
|
||||||
template = encoding.get("value_template")
|
|
||||||
if template:
|
|
||||||
return [(param, str(template).format(value=criterion.get("value"), **mapping))]
|
|
||||||
raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}")
|
|
||||||
|
|
||||||
|
|
||||||
def _encode_range(encoding: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
|
||||||
operator = str(criterion.get("operator") or "<=")
|
|
||||||
value = criterion.get("value")
|
|
||||||
min_param = str(encoding.get("min_parameter") or encoding.get("url_param") or "bpmin")
|
|
||||||
max_param = str(encoding.get("max_parameter") or "bpmax")
|
|
||||||
if operator == "between":
|
|
||||||
if isinstance(value, dict):
|
|
||||||
items: list[tuple[str, str]] = []
|
|
||||||
if value.get("min") is not None:
|
|
||||||
items.append((min_param, str(value["min"])))
|
|
||||||
if value.get("max") is not None:
|
|
||||||
items.append((max_param, str(value["max"])))
|
|
||||||
return items
|
|
||||||
if isinstance(value, list) and len(value) == 2:
|
|
||||||
return [(min_param, str(value[0])), (max_param, str(value[1]))]
|
|
||||||
if operator in {">=", ">"}:
|
|
||||||
return [(min_param, str(value))]
|
|
||||||
return [(max_param, str(value))]
|
|
||||||
|
|
||||||
|
|
||||||
def _encode_range_token(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
|
||||||
filter_key = str(mapping["external_filter_key"])
|
|
||||||
looked_up = _lookup_option(mapping, criterion.get("value"))
|
|
||||||
if looked_up is not None:
|
|
||||||
return [("xf", f"{filter_key}_{_plus(looked_up)}")]
|
|
||||||
operator = str(criterion.get("operator") or "<=")
|
|
||||||
value = criterion.get("value")
|
|
||||||
if operator == "between":
|
|
||||||
if isinstance(value, dict):
|
|
||||||
raw = f"{value.get('min') or ''}*{value.get('max') or ''}"
|
|
||||||
elif isinstance(value, list) and len(value) == 2:
|
|
||||||
raw = f"{value[0]}*{value[1]}"
|
|
||||||
else:
|
|
||||||
raw = f"*{value}"
|
|
||||||
elif operator in {">=", ">"}:
|
|
||||||
raw = f"{value}*"
|
|
||||||
else:
|
|
||||||
raw = f"*{value}"
|
|
||||||
return [("xf", f"{filter_key}_{raw}")]
|
|
||||||
|
|
||||||
|
|
||||||
def _lookup_option(mapping: dict[str, Any], value: Any) -> str | None:
|
|
||||||
options = (mapping.get("encoding") or {}).get("options") or {}
|
|
||||||
if value in options:
|
|
||||||
return str(options[value])
|
|
||||||
if str(value) in options:
|
|
||||||
return str(options[str(value)])
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _encode_xf(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
|
||||||
filter_key = str(mapping["external_filter_key"])
|
|
||||||
raw = criterion.get("value")
|
|
||||||
values = raw if isinstance(raw, list) else [raw]
|
|
||||||
fragments: list[tuple[str, str]] = []
|
|
||||||
for value in values:
|
|
||||||
if value is None or value == "":
|
|
||||||
continue
|
|
||||||
source_value = _xf_value(mapping, {**criterion, "value": value})
|
|
||||||
fragments.append(("xf", f"{filter_key}_{_plus(source_value)}"))
|
|
||||||
if fragments:
|
|
||||||
return fragments
|
|
||||||
source_value = _xf_value(mapping, criterion)
|
|
||||||
return [("xf", f"{filter_key}_{_plus(source_value)}")]
|
|
||||||
|
|
||||||
|
|
||||||
def _xf_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
|
||||||
looked_up = _lookup_option(mapping, criterion.get("value"))
|
|
||||||
if looked_up is not None:
|
|
||||||
return looked_up
|
|
||||||
options = (mapping.get("encoding") or {}).get("options") or {}
|
|
||||||
value = criterion.get("value")
|
|
||||||
if options and value is not None and value != "":
|
|
||||||
return str(value)
|
|
||||||
return str(mapping.get("source_value") or _threshold_value(mapping, criterion))
|
|
||||||
|
|
||||||
|
|
||||||
def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
|
||||||
option = mapping.get("source_value")
|
|
||||||
if option:
|
|
||||||
return str(option)
|
|
||||||
value = criterion.get("value")
|
|
||||||
unit = mapping.get("unit") or ""
|
|
||||||
operator = criterion.get("operator")
|
|
||||||
if operator in {">=", ">"} and unit:
|
|
||||||
return f"ab {value}{unit}"
|
|
||||||
return str(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _plus(value: str) -> str:
|
|
||||||
return value.replace(" ", "+")
|
|
||||||
|
|
||||||
|
|
||||||
def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]:
|
|
||||||
parts = urlsplit(url)
|
|
||||||
decoded: list[tuple[str, str]] = []
|
|
||||||
for name, value in parse_qsl(parts.query, keep_blank_values=True):
|
|
||||||
if name != "xf":
|
|
||||||
continue
|
|
||||||
for part in value.replace("|", "~").split("~"):
|
|
||||||
if "_" not in part:
|
|
||||||
continue
|
|
||||||
key, raw = part.split("_", 1)
|
|
||||||
decoded.append((key, raw.replace("+", " ")))
|
|
||||||
return decoded
|
|
||||||
|
|
||||||
|
|
||||||
def same_origin_url(base: str, maybe_relative: str) -> str:
|
|
||||||
return urljoin(base, maybe_relative)
|
|
||||||
|
|
@ -1,45 +0,0 @@
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Iterable
|
|
||||||
|
|
||||||
from app.domain.enums import NormalizationStatus
|
|
||||||
from app.domain.models import AttributeStatement
|
|
||||||
|
|
||||||
|
|
||||||
def statement_is_usable(statement: AttributeStatement | None, minimum_confidence: Decimal) -> bool:
|
|
||||||
if statement is None:
|
|
||||||
return False
|
|
||||||
if statement.normalization_status != NormalizationStatus.NORMALIZED:
|
|
||||||
return False
|
|
||||||
if statement.normalized_value is None:
|
|
||||||
return False
|
|
||||||
return statement.confidence >= minimum_confidence
|
|
||||||
|
|
||||||
|
|
||||||
def select_attribute_snapshot(
|
|
||||||
statements: Iterable[AttributeStatement],
|
|
||||||
*,
|
|
||||||
as_of: datetime,
|
|
||||||
snapshot_cutoff: datetime,
|
|
||||||
) -> dict[str, AttributeStatement]:
|
|
||||||
"""Pick the newest statement per key at or before the evaluation cutoff."""
|
|
||||||
chosen: dict[str, AttributeStatement] = {}
|
|
||||||
for statement in statements:
|
|
||||||
if statement.observed_at > as_of or statement.ingested_at > snapshot_cutoff:
|
|
||||||
continue
|
|
||||||
current = chosen.get(statement.attribute)
|
|
||||||
if current is None or _is_newer(statement, current):
|
|
||||||
chosen[statement.attribute] = statement
|
|
||||||
return chosen
|
|
||||||
|
|
||||||
|
|
||||||
def _is_newer(candidate: AttributeStatement, current: AttributeStatement) -> bool:
|
|
||||||
return (
|
|
||||||
candidate.observed_at,
|
|
||||||
candidate.ingested_at,
|
|
||||||
candidate.id,
|
|
||||||
) > (
|
|
||||||
current.observed_at,
|
|
||||||
current.ingested_at,
|
|
||||||
current.id,
|
|
||||||
)
|
|
||||||
|
|
@ -1,148 +0,0 @@
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from app.domain.attributes import statement_is_usable
|
|
||||||
from app.domain.enums import ConstraintOperator, ConstraintStatus, ValueType
|
|
||||||
from app.domain.models import AttributeDefinition, AttributeStatement, Constraint, ConstraintResult
|
|
||||||
|
|
||||||
_NUMBER_OPS = {
|
|
||||||
ConstraintOperator.GE,
|
|
||||||
ConstraintOperator.LE,
|
|
||||||
ConstraintOperator.GT,
|
|
||||||
ConstraintOperator.LT,
|
|
||||||
}
|
|
||||||
_EQUALITY_OPS = {ConstraintOperator.EQ, ConstraintOperator.NE}
|
|
||||||
_SCALAR_TYPES = {ValueType.NUMBER, ValueType.STRING, ValueType.BOOLEAN}
|
|
||||||
|
|
||||||
|
|
||||||
def operator_allowed_for(value_type: ValueType, operator: ConstraintOperator) -> bool:
|
|
||||||
if operator == ConstraintOperator.EXISTS:
|
|
||||||
return True
|
|
||||||
if operator in _NUMBER_OPS:
|
|
||||||
return value_type == ValueType.NUMBER
|
|
||||||
if operator in _EQUALITY_OPS:
|
|
||||||
return value_type in _SCALAR_TYPES
|
|
||||||
if operator == ConstraintOperator.IN:
|
|
||||||
return value_type in _SCALAR_TYPES
|
|
||||||
if operator == ConstraintOperator.CONTAINS:
|
|
||||||
return value_type == ValueType.STRING_LIST
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def evaluate_constraint(
|
|
||||||
constraint: Constraint,
|
|
||||||
statement: AttributeStatement | None,
|
|
||||||
definition: AttributeDefinition,
|
|
||||||
minimum_confidence: Decimal,
|
|
||||||
) -> ConstraintResult:
|
|
||||||
usable = statement_is_usable(statement, minimum_confidence)
|
|
||||||
actual = statement.normalized_value if usable else None
|
|
||||||
attribute_row_id = statement.id if statement is not None else None
|
|
||||||
|
|
||||||
if not usable:
|
|
||||||
return ConstraintResult(
|
|
||||||
constraint_id=constraint.id,
|
|
||||||
attribute=constraint.attribute,
|
|
||||||
operator=constraint.operator.value,
|
|
||||||
expected=constraint.value,
|
|
||||||
actual=None,
|
|
||||||
status=ConstraintStatus.UNKNOWN.value,
|
|
||||||
mandatory=constraint.mandatory,
|
|
||||||
attribute_row_id=attribute_row_id,
|
|
||||||
reason_code="ATTRIBUTE_UNUSABLE",
|
|
||||||
)
|
|
||||||
|
|
||||||
passed = _compare(constraint.operator, actual, constraint.value, definition.value_type)
|
|
||||||
if passed is None:
|
|
||||||
status = ConstraintStatus.UNKNOWN
|
|
||||||
reason = "TYPE_MISMATCH"
|
|
||||||
elif passed:
|
|
||||||
status = ConstraintStatus.PASS
|
|
||||||
reason = "CONSTRAINT_PASS"
|
|
||||||
else:
|
|
||||||
status = ConstraintStatus.FAIL
|
|
||||||
reason = "CONSTRAINT_FAIL"
|
|
||||||
return ConstraintResult(
|
|
||||||
constraint_id=constraint.id,
|
|
||||||
attribute=constraint.attribute,
|
|
||||||
operator=constraint.operator.value,
|
|
||||||
expected=constraint.value,
|
|
||||||
actual=actual,
|
|
||||||
status=status.value,
|
|
||||||
mandatory=constraint.mandatory,
|
|
||||||
attribute_row_id=attribute_row_id,
|
|
||||||
reason_code=reason,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _as_number(value: Any) -> Decimal | None:
|
|
||||||
if isinstance(value, bool) or value is None:
|
|
||||||
return None
|
|
||||||
if isinstance(value, Decimal):
|
|
||||||
return value
|
|
||||||
if isinstance(value, int):
|
|
||||||
return Decimal(value)
|
|
||||||
if isinstance(value, float):
|
|
||||||
return Decimal(str(value))
|
|
||||||
if isinstance(value, str):
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _same_scalar(left: Any, right: Any, value_type: ValueType) -> bool | None:
|
|
||||||
if value_type == ValueType.NUMBER:
|
|
||||||
left_n = _as_number(left)
|
|
||||||
right_n = _as_number(right)
|
|
||||||
if left_n is None or right_n is None:
|
|
||||||
return None
|
|
||||||
return left_n == right_n
|
|
||||||
if value_type == ValueType.STRING:
|
|
||||||
if not isinstance(left, str) or not isinstance(right, str):
|
|
||||||
return None
|
|
||||||
return left == right
|
|
||||||
if value_type == ValueType.BOOLEAN:
|
|
||||||
if not isinstance(left, bool) or not isinstance(right, bool):
|
|
||||||
return None
|
|
||||||
return left == right
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _compare(
|
|
||||||
operator: ConstraintOperator,
|
|
||||||
actual: Any,
|
|
||||||
expected: Any,
|
|
||||||
value_type: ValueType,
|
|
||||||
) -> bool | None:
|
|
||||||
if operator == ConstraintOperator.EXISTS:
|
|
||||||
return True
|
|
||||||
if operator in _NUMBER_OPS:
|
|
||||||
actual_n = _as_number(actual)
|
|
||||||
expected_n = _as_number(expected)
|
|
||||||
if actual_n is None or expected_n is None:
|
|
||||||
return None
|
|
||||||
if operator == ConstraintOperator.GE:
|
|
||||||
return actual_n >= expected_n
|
|
||||||
if operator == ConstraintOperator.LE:
|
|
||||||
return actual_n <= expected_n
|
|
||||||
if operator == ConstraintOperator.GT:
|
|
||||||
return actual_n > expected_n
|
|
||||||
return actual_n < expected_n
|
|
||||||
if operator == ConstraintOperator.EQ:
|
|
||||||
return _same_scalar(actual, expected, value_type)
|
|
||||||
if operator == ConstraintOperator.NE:
|
|
||||||
equal = _same_scalar(actual, expected, value_type)
|
|
||||||
return None if equal is None else (not equal)
|
|
||||||
if operator == ConstraintOperator.IN:
|
|
||||||
if not isinstance(expected, list) or not expected:
|
|
||||||
return None
|
|
||||||
matches = [_same_scalar(actual, item, value_type) for item in expected]
|
|
||||||
if any(item is None for item in matches):
|
|
||||||
return None
|
|
||||||
return any(matches)
|
|
||||||
if operator == ConstraintOperator.CONTAINS:
|
|
||||||
if not isinstance(actual, list) or not isinstance(expected, str):
|
|
||||||
return None
|
|
||||||
if any(not isinstance(item, str) for item in actual):
|
|
||||||
return None
|
|
||||||
return expected in actual
|
|
||||||
return None
|
|
||||||
|
|
@ -1,81 +0,0 @@
|
||||||
from enum import StrEnum
|
|
||||||
|
|
||||||
|
|
||||||
class ValueType(StrEnum):
|
|
||||||
NUMBER = "number"
|
|
||||||
STRING = "string"
|
|
||||||
BOOLEAN = "boolean"
|
|
||||||
STRING_LIST = "string_list"
|
|
||||||
|
|
||||||
|
|
||||||
class OfferCondition(StrEnum):
|
|
||||||
NEW = "new"
|
|
||||||
REFURBISHED = "refurbished"
|
|
||||||
USED = "used"
|
|
||||||
|
|
||||||
|
|
||||||
class Availability(StrEnum):
|
|
||||||
IN_STOCK = "in_stock"
|
|
||||||
OUT_OF_STOCK = "out_of_stock"
|
|
||||||
UNKNOWN = "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
class NormalizationStatus(StrEnum):
|
|
||||||
NORMALIZED = "normalized"
|
|
||||||
UNKNOWN = "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
class ConstraintStatus(StrEnum):
|
|
||||||
PASS = "PASS"
|
|
||||||
FAIL = "FAIL"
|
|
||||||
UNKNOWN = "UNKNOWN"
|
|
||||||
|
|
||||||
|
|
||||||
class Eligibility(StrEnum):
|
|
||||||
ELIGIBLE = "ELIGIBLE"
|
|
||||||
INELIGIBLE = "INELIGIBLE"
|
|
||||||
UNKNOWN = "UNKNOWN"
|
|
||||||
|
|
||||||
|
|
||||||
class PreferenceStrategy(StrEnum):
|
|
||||||
NUMERIC_LINEAR = "numeric_linear"
|
|
||||||
|
|
||||||
|
|
||||||
class PreferenceDirection(StrEnum):
|
|
||||||
HIGHER = "higher"
|
|
||||||
LOWER = "lower"
|
|
||||||
|
|
||||||
|
|
||||||
class ConstraintOperator(StrEnum):
|
|
||||||
GE = ">="
|
|
||||||
LE = "<="
|
|
||||||
GT = ">"
|
|
||||||
LT = "<"
|
|
||||||
EQ = "=="
|
|
||||||
NE = "!="
|
|
||||||
IN = "in"
|
|
||||||
CONTAINS = "contains"
|
|
||||||
EXISTS = "exists"
|
|
||||||
|
|
||||||
|
|
||||||
class Classification(StrEnum):
|
|
||||||
IGNORE = "IGNORE"
|
|
||||||
WATCH = "WATCH"
|
|
||||||
GOOD = "GOOD"
|
|
||||||
VERY_GOOD = "VERY_GOOD"
|
|
||||||
EXCEPTIONAL = "EXCEPTIONAL"
|
|
||||||
INELIGIBLE = "INELIGIBLE"
|
|
||||||
INSUFFICIENT_DATA = "INSUFFICIENT_DATA"
|
|
||||||
|
|
||||||
|
|
||||||
class PriceBasis(StrEnum):
|
|
||||||
BUDGET_ONLY = "BUDGET_ONLY"
|
|
||||||
BUDGET_AND_HISTORY = "BUDGET_AND_HISTORY"
|
|
||||||
|
|
||||||
|
|
||||||
class PriceReferenceScope(StrEnum):
|
|
||||||
OFFER = "offer"
|
|
||||||
|
|
||||||
|
|
||||||
class AdapterType(StrEnum):
|
|
||||||
MANUAL = "manual"
|
|
||||||
|
|
@ -1,275 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from app.domain.enums import (
|
|
||||||
Availability,
|
|
||||||
ConstraintOperator,
|
|
||||||
Eligibility,
|
|
||||||
NormalizationStatus,
|
|
||||||
OfferCondition,
|
|
||||||
PreferenceDirection,
|
|
||||||
PreferenceStrategy,
|
|
||||||
PriceBasis,
|
|
||||||
PriceReferenceScope,
|
|
||||||
ValueType,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Budget:
|
|
||||||
target: Decimal
|
|
||||||
maximum: Decimal
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Constraint:
|
|
||||||
id: str
|
|
||||||
attribute: str
|
|
||||||
operator: ConstraintOperator
|
|
||||||
value: Any = None
|
|
||||||
mandatory: bool = True
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Preference:
|
|
||||||
id: str
|
|
||||||
attribute: str
|
|
||||||
strategy: PreferenceStrategy
|
|
||||||
direction: PreferenceDirection
|
|
||||||
acceptable: Decimal
|
|
||||||
ideal: Decimal
|
|
||||||
weight: Decimal
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ClassificationBound:
|
|
||||||
upper: Decimal
|
|
||||||
inclusive: bool
|
|
||||||
label: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ScoringModel:
|
|
||||||
version: str
|
|
||||||
algorithm_id: str
|
|
||||||
supported_preference_strategies: tuple[str, ...]
|
|
||||||
minimum_confidence: Decimal
|
|
||||||
maximum_price_age_hours: int
|
|
||||||
optional_constraint_penalty_points: Decimal
|
|
||||||
missing_preference_score: Decimal
|
|
||||||
product_weight: Decimal
|
|
||||||
price_weight: Decimal
|
|
||||||
budget_weight: Decimal
|
|
||||||
history_weight: Decimal
|
|
||||||
history_window_days: int
|
|
||||||
minimum_history_days: int
|
|
||||||
history_discount_for_max_score: Decimal
|
|
||||||
classification_upper_bounds: tuple[ClassificationBound, ...]
|
|
||||||
definition: dict[str, Any]
|
|
||||||
definition_hash: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Profile:
|
|
||||||
id: str
|
|
||||||
version: int
|
|
||||||
name: str
|
|
||||||
category: str
|
|
||||||
currency: str
|
|
||||||
market: str
|
|
||||||
allowed_conditions: tuple[OfferCondition, ...]
|
|
||||||
budget: Budget
|
|
||||||
scoring_version: str
|
|
||||||
constraints: tuple[Constraint, ...]
|
|
||||||
preferences: tuple[Preference, ...]
|
|
||||||
definition: dict[str, Any]
|
|
||||||
definition_hash: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class AttributeDefinition:
|
|
||||||
key: str
|
|
||||||
value_type: ValueType
|
|
||||||
canonical_unit: str | None
|
|
||||||
minimum: Decimal | None
|
|
||||||
maximum: Decimal | None
|
|
||||||
description: str | None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Product:
|
|
||||||
id: UUID
|
|
||||||
category: str
|
|
||||||
brand: str
|
|
||||||
name: str
|
|
||||||
model: str | None
|
|
||||||
created_at: datetime
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ProductVariant:
|
|
||||||
id: UUID
|
|
||||||
product_id: UUID
|
|
||||||
name: str
|
|
||||||
manufacturer_sku: str | None
|
|
||||||
identifiers: dict[str, Any]
|
|
||||||
created_at: datetime
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class AttributeStatement:
|
|
||||||
id: UUID
|
|
||||||
variant_id: UUID
|
|
||||||
attribute: str
|
|
||||||
raw_value: Any
|
|
||||||
normalized_value: Any
|
|
||||||
unit: str | None
|
|
||||||
normalization_status: NormalizationStatus
|
|
||||||
normalizer_version: str
|
|
||||||
confidence: Decimal
|
|
||||||
source_id: str
|
|
||||||
source_reference: str
|
|
||||||
observed_at: datetime
|
|
||||||
ingested_at: datetime
|
|
||||||
normalization_error: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Offer:
|
|
||||||
id: UUID
|
|
||||||
variant_id: UUID
|
|
||||||
source_id: str
|
|
||||||
external_id: str
|
|
||||||
seller: str
|
|
||||||
url: str | None
|
|
||||||
market: str
|
|
||||||
currency: str
|
|
||||||
condition: OfferCondition
|
|
||||||
created_at: datetime
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class PriceObservation:
|
|
||||||
id: UUID
|
|
||||||
offer_id: UUID
|
|
||||||
source_id: str
|
|
||||||
external_observation_id: str
|
|
||||||
observed_at: datetime
|
|
||||||
ingested_at: datetime
|
|
||||||
item_price: Decimal
|
|
||||||
shipping_price: Decimal | None
|
|
||||||
currency: str
|
|
||||||
availability: Availability
|
|
||||||
raw_payload: Any
|
|
||||||
confidence: Decimal
|
|
||||||
|
|
||||||
@property
|
|
||||||
def total_price(self) -> Decimal | None:
|
|
||||||
if self.shipping_price is None:
|
|
||||||
return None
|
|
||||||
return self.item_price + self.shipping_price
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ConstraintResult:
|
|
||||||
constraint_id: str
|
|
||||||
attribute: str
|
|
||||||
operator: str
|
|
||||||
expected: Any
|
|
||||||
actual: Any
|
|
||||||
status: str
|
|
||||||
mandatory: bool
|
|
||||||
attribute_row_id: UUID | None
|
|
||||||
reason_code: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class GateResult:
|
|
||||||
key: str
|
|
||||||
status: str
|
|
||||||
reason_code: str
|
|
||||||
details: dict[str, Any] = field(default_factory=dict)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ScoreComponent:
|
|
||||||
kind: str
|
|
||||||
component_key: str
|
|
||||||
raw_score: Decimal | None
|
|
||||||
weight: Decimal | None
|
|
||||||
contribution: Decimal | None
|
|
||||||
details: dict[str, Any]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class HistoryWindow:
|
|
||||||
days: int
|
|
||||||
values: tuple[Decimal, ...]
|
|
||||||
observation_ids: tuple[UUID, ...]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def day_count(self) -> int:
|
|
||||||
return len(self.values)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class PriceStatistics:
|
|
||||||
current_total: Decimal | None
|
|
||||||
median_30d: Decimal | None
|
|
||||||
median_30d_days: int
|
|
||||||
median_90d: Decimal | None
|
|
||||||
median_90d_days: int
|
|
||||||
lowest_365d: Decimal | None
|
|
||||||
lowest_365d_days: int
|
|
||||||
lowest_365d_is_full_year: bool
|
|
||||||
used_observation_ids: tuple[UUID, ...]
|
|
||||||
current_observation_id: UUID | None
|
|
||||||
price_reference_scope: PriceReferenceScope = PriceReferenceScope.OFFER
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Reason:
|
|
||||||
code: str
|
|
||||||
message: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class EvaluationResult:
|
|
||||||
profile_id: str
|
|
||||||
profile_version: int
|
|
||||||
scoring_version: str
|
|
||||||
scoring_hash: str
|
|
||||||
engine_build: str
|
|
||||||
variant_id: UUID
|
|
||||||
offer_id: UUID
|
|
||||||
as_of: datetime
|
|
||||||
snapshot_cutoff: datetime
|
|
||||||
product_eligibility: Eligibility
|
|
||||||
offer_eligibility: Eligibility
|
|
||||||
overall_eligibility: Eligibility
|
|
||||||
eligible: bool | None
|
|
||||||
product_score: Decimal | None
|
|
||||||
product_score_raw: Decimal | None
|
|
||||||
price_score: Decimal | None
|
|
||||||
price_score_raw: Decimal | None
|
|
||||||
deal_score: Decimal | None
|
|
||||||
deal_score_raw: Decimal | None
|
|
||||||
classification: str
|
|
||||||
price_basis: PriceBasis | None
|
|
||||||
product_coverage: Decimal
|
|
||||||
constraint_results: tuple[ConstraintResult, ...]
|
|
||||||
product_gates: tuple[GateResult, ...]
|
|
||||||
offer_gates: tuple[GateResult, ...]
|
|
||||||
components: tuple[ScoreComponent, ...]
|
|
||||||
price_statistics: PriceStatistics
|
|
||||||
price_reference_scope: PriceReferenceScope
|
|
||||||
reasons: tuple[Reason, ...]
|
|
||||||
warnings: tuple[str, ...]
|
|
||||||
input_snapshot: dict[str, Any]
|
|
||||||
failed_constraints: tuple[ConstraintResult, ...]
|
|
||||||
unknown_constraints: tuple[ConstraintResult, ...]
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
from datetime import date, datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from app.domain.timeutil import to_iso_z
|
|
||||||
|
|
||||||
|
|
||||||
def jsonable(value: Any) -> Any:
|
|
||||||
if value is None or isinstance(value, (str, int, bool)):
|
|
||||||
return value
|
|
||||||
if isinstance(value, Decimal):
|
|
||||||
return format(value, "f")
|
|
||||||
if isinstance(value, UUID):
|
|
||||||
return str(value)
|
|
||||||
if isinstance(value, datetime):
|
|
||||||
return to_iso_z(value)
|
|
||||||
if isinstance(value, date):
|
|
||||||
return value.isoformat()
|
|
||||||
if isinstance(value, dict):
|
|
||||||
return {str(key): jsonable(item) for key, item in value.items()}
|
|
||||||
if isinstance(value, (list, tuple)):
|
|
||||||
return [jsonable(item) for item in value]
|
|
||||||
if hasattr(value, "value"):
|
|
||||||
return jsonable(value.value)
|
|
||||||
return value
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
|
|
||||||
def require_utc(value: datetime, field_name: str) -> datetime:
|
|
||||||
if value.tzinfo is None:
|
|
||||||
raise ValueError(f"{field_name} must include a timezone")
|
|
||||||
return value.astimezone(timezone.utc)
|
|
||||||
|
|
||||||
|
|
||||||
def to_iso_z(value: datetime) -> str:
|
|
||||||
return require_utc(value, "datetime").strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
||||||
|
|
@ -1,307 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from app.domain.constraints import operator_allowed_for
|
|
||||||
from app.domain.enums import (
|
|
||||||
ConstraintOperator,
|
|
||||||
OfferCondition,
|
|
||||||
PreferenceDirection,
|
|
||||||
PreferenceStrategy,
|
|
||||||
ValueType,
|
|
||||||
)
|
|
||||||
from app.domain.models import (
|
|
||||||
AttributeDefinition,
|
|
||||||
Budget,
|
|
||||||
ClassificationBound,
|
|
||||||
Constraint,
|
|
||||||
Preference,
|
|
||||||
Profile,
|
|
||||||
ScoringModel,
|
|
||||||
)
|
|
||||||
|
|
||||||
SNAKE_CASE = r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$"
|
|
||||||
CURRENCY = r"^[A-Z]{3}$"
|
|
||||||
SUPPORTED_ALGORITHM = "linear-budget-history-v1"
|
|
||||||
|
|
||||||
|
|
||||||
class DomainValidationError(ValueError):
|
|
||||||
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
|
|
||||||
super().__init__(message)
|
|
||||||
self.details = details or {}
|
|
||||||
|
|
||||||
|
|
||||||
def canonical_json(data: Any) -> str:
|
|
||||||
return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
||||||
|
|
||||||
|
|
||||||
def sha256_text(text: str) -> str:
|
|
||||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _decimal(value: Any, field: str) -> Decimal:
|
|
||||||
if isinstance(value, bool) or value is None:
|
|
||||||
raise DomainValidationError(f"{field} must be a decimal", {"field": field})
|
|
||||||
try:
|
|
||||||
return Decimal(str(value))
|
|
||||||
except Exception as exc:
|
|
||||||
raise DomainValidationError(f"{field} must be a decimal", {"field": field}) from exc
|
|
||||||
|
|
||||||
|
|
||||||
def _require(mapping: dict[str, Any], key: str) -> Any:
|
|
||||||
if key not in mapping:
|
|
||||||
raise DomainValidationError(f"missing field {key}", {"field": key})
|
|
||||||
return mapping[key]
|
|
||||||
|
|
||||||
|
|
||||||
def validate_attribute_key(key: str) -> None:
|
|
||||||
import re
|
|
||||||
|
|
||||||
if not re.fullmatch(SNAKE_CASE, key):
|
|
||||||
raise DomainValidationError("attribute key must be snake_case", {"key": key})
|
|
||||||
|
|
||||||
|
|
||||||
def validate_currency(value: str) -> str:
|
|
||||||
import re
|
|
||||||
|
|
||||||
if not re.fullmatch(CURRENCY, value):
|
|
||||||
raise DomainValidationError("currency must be a 3-letter uppercase code", {"currency": value})
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def parse_scoring_model(definition: dict[str, Any]) -> ScoringModel:
|
|
||||||
version = str(_require(definition, "version"))
|
|
||||||
algorithm_id = str(_require(definition, "algorithm_id"))
|
|
||||||
if algorithm_id != SUPPORTED_ALGORITHM:
|
|
||||||
raise DomainValidationError("unsupported algorithm_id", {"algorithm_id": algorithm_id})
|
|
||||||
strategies = tuple(str(item) for item in _require(definition, "supported_preference_strategies"))
|
|
||||||
if not strategies:
|
|
||||||
raise DomainValidationError("supported_preference_strategies must not be empty")
|
|
||||||
if PreferenceStrategy.NUMERIC_LINEAR.value not in strategies:
|
|
||||||
raise DomainValidationError("generic-v1 must support numeric_linear")
|
|
||||||
|
|
||||||
min_conf = _decimal(_require(definition, "minimum_confidence"), "minimum_confidence")
|
|
||||||
if not Decimal("0") <= min_conf <= Decimal("1"):
|
|
||||||
raise DomainValidationError("minimum_confidence must be between 0 and 1")
|
|
||||||
|
|
||||||
max_age = int(_require(definition, "maximum_price_age_hours"))
|
|
||||||
history_window = int(_require(definition, "history_window_days"))
|
|
||||||
min_history = int(_require(definition, "minimum_history_days"))
|
|
||||||
if max_age <= 0 or history_window <= 0 or min_history <= 0:
|
|
||||||
raise DomainValidationError("time windows must be positive")
|
|
||||||
|
|
||||||
product_weight = _decimal(_require(definition, "product_weight"), "product_weight")
|
|
||||||
price_weight = _decimal(_require(definition, "price_weight"), "price_weight")
|
|
||||||
budget_weight = _decimal(_require(definition, "budget_weight"), "budget_weight")
|
|
||||||
history_weight = _decimal(_require(definition, "history_weight"), "history_weight")
|
|
||||||
for name, weight in (
|
|
||||||
("product_weight", product_weight),
|
|
||||||
("price_weight", price_weight),
|
|
||||||
("budget_weight", budget_weight),
|
|
||||||
("history_weight", history_weight),
|
|
||||||
):
|
|
||||||
if weight < 0:
|
|
||||||
raise DomainValidationError(f"{name} must be non-negative")
|
|
||||||
if product_weight + price_weight != Decimal("1"):
|
|
||||||
raise DomainValidationError("product_weight + price_weight must equal 1")
|
|
||||||
if budget_weight + history_weight != Decimal("1"):
|
|
||||||
raise DomainValidationError("budget_weight + history_weight must equal 1")
|
|
||||||
|
|
||||||
bounds_raw = _require(definition, "classification_upper_bounds")
|
|
||||||
bounds: list[ClassificationBound] = []
|
|
||||||
previous = Decimal("0")
|
|
||||||
for index, raw in enumerate(bounds_raw):
|
|
||||||
upper = _decimal(raw["upper"], f"classification_upper_bounds[{index}].upper")
|
|
||||||
inclusive = bool(raw["inclusive"])
|
|
||||||
label = str(raw["label"])
|
|
||||||
if upper < previous or (index and upper == previous):
|
|
||||||
raise DomainValidationError("classification bounds must be strictly ascending")
|
|
||||||
if index == 0 and upper <= 0:
|
|
||||||
raise DomainValidationError("classification bounds must start above 0")
|
|
||||||
bounds.append(ClassificationBound(upper=upper, inclusive=inclusive, label=label))
|
|
||||||
previous = upper
|
|
||||||
if not bounds or bounds[-1].upper != Decimal("100"):
|
|
||||||
raise DomainValidationError("classification bounds must end at 100")
|
|
||||||
|
|
||||||
canonical = json.loads(canonical_json(definition))
|
|
||||||
return ScoringModel(
|
|
||||||
version=version,
|
|
||||||
algorithm_id=algorithm_id,
|
|
||||||
supported_preference_strategies=strategies,
|
|
||||||
minimum_confidence=min_conf,
|
|
||||||
maximum_price_age_hours=max_age,
|
|
||||||
optional_constraint_penalty_points=_decimal(
|
|
||||||
_require(definition, "optional_constraint_penalty_points"),
|
|
||||||
"optional_constraint_penalty_points",
|
|
||||||
),
|
|
||||||
missing_preference_score=_decimal(
|
|
||||||
_require(definition, "missing_preference_score"),
|
|
||||||
"missing_preference_score",
|
|
||||||
),
|
|
||||||
product_weight=product_weight,
|
|
||||||
price_weight=price_weight,
|
|
||||||
budget_weight=budget_weight,
|
|
||||||
history_weight=history_weight,
|
|
||||||
history_window_days=history_window,
|
|
||||||
minimum_history_days=min_history,
|
|
||||||
history_discount_for_max_score=_decimal(
|
|
||||||
_require(definition, "history_discount_for_max_score"),
|
|
||||||
"history_discount_for_max_score",
|
|
||||||
),
|
|
||||||
classification_upper_bounds=tuple(bounds),
|
|
||||||
definition=canonical,
|
|
||||||
definition_hash=sha256_text(canonical_json(canonical)),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_profile(
|
|
||||||
definition: dict[str, Any],
|
|
||||||
*,
|
|
||||||
attribute_definitions: dict[str, AttributeDefinition],
|
|
||||||
scoring_versions: set[str],
|
|
||||||
supported_strategies: set[str],
|
|
||||||
) -> Profile:
|
|
||||||
profile_id = str(_require(definition, "id"))
|
|
||||||
version = int(_require(definition, "version"))
|
|
||||||
if version <= 0:
|
|
||||||
raise DomainValidationError("profile version must be > 0")
|
|
||||||
name = str(_require(definition, "name"))
|
|
||||||
category = str(_require(definition, "category"))
|
|
||||||
if not category:
|
|
||||||
raise DomainValidationError("category is required")
|
|
||||||
currency = validate_currency(str(_require(definition, "currency")))
|
|
||||||
market = str(_require(definition, "market"))
|
|
||||||
if not market:
|
|
||||||
raise DomainValidationError("market is required")
|
|
||||||
raw_conditions = _require(definition, "allowed_conditions")
|
|
||||||
if not raw_conditions:
|
|
||||||
raise DomainValidationError("allowed_conditions is required")
|
|
||||||
conditions = tuple(OfferCondition(item) for item in raw_conditions)
|
|
||||||
scoring_version = str(_require(definition, "scoring_version"))
|
|
||||||
if scoring_version not in scoring_versions:
|
|
||||||
raise DomainValidationError("unknown scoring_version", {"scoring_version": scoring_version})
|
|
||||||
|
|
||||||
budget_raw = _require(definition, "budget")
|
|
||||||
target = _decimal(_require(budget_raw, "target"), "budget.target")
|
|
||||||
maximum = _decimal(_require(budget_raw, "maximum"), "budget.maximum")
|
|
||||||
if not (Decimal("0") < target < maximum):
|
|
||||||
raise DomainValidationError("budget must satisfy 0 < target < maximum")
|
|
||||||
|
|
||||||
constraints = _parse_constraints(_require(definition, "constraints"), attribute_definitions)
|
|
||||||
preferences = _parse_preferences(
|
|
||||||
_require(definition, "preferences"),
|
|
||||||
attribute_definitions,
|
|
||||||
supported_strategies,
|
|
||||||
)
|
|
||||||
canonical = json.loads(canonical_json(definition))
|
|
||||||
return Profile(
|
|
||||||
id=profile_id,
|
|
||||||
version=version,
|
|
||||||
name=name,
|
|
||||||
category=category,
|
|
||||||
currency=currency,
|
|
||||||
market=market,
|
|
||||||
allowed_conditions=conditions,
|
|
||||||
budget=Budget(target=target, maximum=maximum),
|
|
||||||
scoring_version=scoring_version,
|
|
||||||
constraints=constraints,
|
|
||||||
preferences=preferences,
|
|
||||||
definition=canonical,
|
|
||||||
definition_hash=sha256_text(canonical_json(canonical)),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_constraints(
|
|
||||||
raw_items: list[dict[str, Any]],
|
|
||||||
attribute_definitions: dict[str, AttributeDefinition],
|
|
||||||
) -> tuple[Constraint, ...]:
|
|
||||||
seen: set[str] = set()
|
|
||||||
items: list[Constraint] = []
|
|
||||||
for raw in raw_items:
|
|
||||||
constraint_id = str(_require(raw, "id"))
|
|
||||||
if constraint_id in seen:
|
|
||||||
raise DomainValidationError("duplicate constraint id", {"id": constraint_id})
|
|
||||||
seen.add(constraint_id)
|
|
||||||
attribute = str(_require(raw, "attribute"))
|
|
||||||
definition = attribute_definitions.get(attribute)
|
|
||||||
if definition is None:
|
|
||||||
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
|
|
||||||
operator = ConstraintOperator(str(_require(raw, "operator")))
|
|
||||||
if not operator_allowed_for(definition.value_type, operator):
|
|
||||||
raise DomainValidationError(
|
|
||||||
"operator is not valid for attribute type",
|
|
||||||
{"attribute": attribute, "operator": operator.value, "value_type": definition.value_type.value},
|
|
||||||
)
|
|
||||||
value = raw.get("value")
|
|
||||||
if operator == ConstraintOperator.EXISTS:
|
|
||||||
if "value" in raw and raw["value"] is not None:
|
|
||||||
raise DomainValidationError("exists must not have a value", {"id": constraint_id})
|
|
||||||
value = None
|
|
||||||
elif operator == ConstraintOperator.IN:
|
|
||||||
if not isinstance(value, list) or not value:
|
|
||||||
raise DomainValidationError("in requires a non-empty list", {"id": constraint_id})
|
|
||||||
elif operator == ConstraintOperator.CONTAINS:
|
|
||||||
if not isinstance(value, str):
|
|
||||||
raise DomainValidationError("contains requires a string", {"id": constraint_id})
|
|
||||||
items.append(
|
|
||||||
Constraint(
|
|
||||||
id=constraint_id,
|
|
||||||
attribute=attribute,
|
|
||||||
operator=operator,
|
|
||||||
value=value,
|
|
||||||
mandatory=bool(raw.get("mandatory", True)),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return tuple(items)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_preferences(
|
|
||||||
raw_items: list[dict[str, Any]],
|
|
||||||
attribute_definitions: dict[str, AttributeDefinition],
|
|
||||||
supported_strategies: set[str],
|
|
||||||
) -> tuple[Preference, ...]:
|
|
||||||
if not raw_items:
|
|
||||||
raise DomainValidationError("at least one preference is required")
|
|
||||||
seen: set[str] = set()
|
|
||||||
items: list[Preference] = []
|
|
||||||
for raw in raw_items:
|
|
||||||
preference_id = str(_require(raw, "id"))
|
|
||||||
if preference_id in seen:
|
|
||||||
raise DomainValidationError("duplicate preference id", {"id": preference_id})
|
|
||||||
seen.add(preference_id)
|
|
||||||
attribute = str(_require(raw, "attribute"))
|
|
||||||
definition = attribute_definitions.get(attribute)
|
|
||||||
if definition is None:
|
|
||||||
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
|
|
||||||
strategy = str(_require(raw, "strategy"))
|
|
||||||
if strategy not in supported_strategies:
|
|
||||||
raise DomainValidationError("unsupported preference strategy", {"strategy": strategy})
|
|
||||||
if strategy != PreferenceStrategy.NUMERIC_LINEAR.value:
|
|
||||||
raise DomainValidationError("v0.1 only supports numeric_linear", {"strategy": strategy})
|
|
||||||
if definition.value_type != ValueType.NUMBER:
|
|
||||||
raise DomainValidationError("numeric_linear requires a number attribute", {"attribute": attribute})
|
|
||||||
direction = PreferenceDirection(str(_require(raw, "direction")))
|
|
||||||
acceptable = _decimal(_require(raw, "acceptable"), "acceptable")
|
|
||||||
ideal = _decimal(_require(raw, "ideal"), "ideal")
|
|
||||||
weight = _decimal(_require(raw, "weight"), "weight")
|
|
||||||
if weight <= 0:
|
|
||||||
raise DomainValidationError("preference weight must be positive", {"id": preference_id})
|
|
||||||
if direction == PreferenceDirection.HIGHER and not ideal > acceptable:
|
|
||||||
raise DomainValidationError("higher requires ideal > acceptable", {"id": preference_id})
|
|
||||||
if direction == PreferenceDirection.LOWER and not ideal < acceptable:
|
|
||||||
raise DomainValidationError("lower requires ideal < acceptable", {"id": preference_id})
|
|
||||||
items.append(
|
|
||||||
Preference(
|
|
||||||
id=preference_id,
|
|
||||||
attribute=attribute,
|
|
||||||
strategy=PreferenceStrategy.NUMERIC_LINEAR,
|
|
||||||
direction=direction,
|
|
||||||
acceptable=acceptable,
|
|
||||||
ideal=ideal,
|
|
||||||
weight=weight,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return tuple(items)
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
class AppError(Exception):
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
status_code: int,
|
|
||||||
code: str,
|
|
||||||
message: str,
|
|
||||||
details: dict[str, Any] | None = None,
|
|
||||||
) -> None:
|
|
||||||
super().__init__(message)
|
|
||||||
self.status_code = status_code
|
|
||||||
self.code = code
|
|
||||||
self.message = message
|
|
||||||
self.details = details or {}
|
|
||||||
|
|
||||||
|
|
||||||
def error_body(code: str, message: str, details: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
||||||
return {"error": {"code": code, "message": message, "details": details or {}}}
|
|
||||||
|
|
@ -1,48 +0,0 @@
|
||||||
from fastapi import FastAPI, Request
|
|
||||||
from fastapi.exceptions import RequestValidationError
|
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
|
||||||
from fastapi.responses import JSONResponse
|
|
||||||
|
|
||||||
from app.api.discovery import router as discovery_router
|
|
||||||
from app.api.evaluations import router as evaluations_router
|
|
||||||
from app.api.health import router as health_router
|
|
||||||
from app.api.offers import router as offers_router
|
|
||||||
from app.api.products import router as products_router
|
|
||||||
from app.api.profiles import router as profiles_router
|
|
||||||
from app.api.projects import router as projects_router
|
|
||||||
from app.errors import AppError, error_body
|
|
||||||
from app.settings import get_settings
|
|
||||||
|
|
||||||
settings = get_settings()
|
|
||||||
app = FastAPI(
|
|
||||||
title="Buying Agent",
|
|
||||||
version="0.1.0",
|
|
||||||
description="Generic buying and deal evaluation API. The backend is the system of record; no frontend or n8n is required.",
|
|
||||||
)
|
|
||||||
app.add_middleware(
|
|
||||||
CORSMiddleware,
|
|
||||||
allow_origins=settings.cors_origins,
|
|
||||||
allow_credentials=True,
|
|
||||||
allow_methods=["*"],
|
|
||||||
allow_headers=["*"],
|
|
||||||
)
|
|
||||||
app.include_router(health_router)
|
|
||||||
app.include_router(profiles_router)
|
|
||||||
app.include_router(products_router)
|
|
||||||
app.include_router(offers_router)
|
|
||||||
app.include_router(evaluations_router)
|
|
||||||
app.include_router(projects_router)
|
|
||||||
app.include_router(discovery_router)
|
|
||||||
|
|
||||||
|
|
||||||
@app.exception_handler(AppError)
|
|
||||||
def handle_app_error(_request: Request, exc: AppError) -> JSONResponse:
|
|
||||||
return JSONResponse(error_body(exc.code, exc.message, exc.details), status_code=exc.status_code)
|
|
||||||
|
|
||||||
|
|
||||||
@app.exception_handler(RequestValidationError)
|
|
||||||
def handle_validation_error(_request: Request, exc: RequestValidationError) -> JSONResponse:
|
|
||||||
return JSONResponse(
|
|
||||||
error_body("VALIDATION_ERROR", "request validation failed", {"errors": exc.errors()}),
|
|
||||||
status_code=422,
|
|
||||||
)
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
from typing import Protocol
|
|
||||||
|
|
||||||
from app.domain.models import AttributeDefinition, AttributeStatement
|
|
||||||
|
|
||||||
|
|
||||||
class Normalizer(Protocol):
|
|
||||||
version: str
|
|
||||||
|
|
||||||
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
|
|
||||||
"""Validate an already normalized statement. Must not parse free text."""
|
|
||||||
...
|
|
||||||
|
|
@ -1,61 +0,0 @@
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from app.domain.enums import NormalizationStatus, ValueType
|
|
||||||
from app.domain.models import AttributeDefinition, AttributeStatement
|
|
||||||
from app.errors import AppError
|
|
||||||
|
|
||||||
|
|
||||||
class ManualNormalizer:
|
|
||||||
version = "manual-v1"
|
|
||||||
|
|
||||||
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
|
|
||||||
if statement.normalization_status == NormalizationStatus.UNKNOWN:
|
|
||||||
if statement.normalized_value is not None:
|
|
||||||
raise AppError(
|
|
||||||
422,
|
|
||||||
"NORMALIZATION_INCONSISTENT",
|
|
||||||
"unknown statements must not have a normalized_value",
|
|
||||||
)
|
|
||||||
return statement
|
|
||||||
if statement.normalization_status != NormalizationStatus.NORMALIZED:
|
|
||||||
raise AppError(422, "NORMALIZATION_STATUS", "unsupported normalization_status")
|
|
||||||
if statement.normalized_value is None:
|
|
||||||
raise AppError(422, "NORMALIZATION_MISSING", "normalized_value is required")
|
|
||||||
_assert_type(definition, statement.normalized_value)
|
|
||||||
if definition.value_type == ValueType.NUMBER:
|
|
||||||
number = Decimal(str(statement.normalized_value))
|
|
||||||
if definition.key == "weight_kg" and number <= 0:
|
|
||||||
raise AppError(422, "VALUE_RANGE", "weight_kg must be > 0")
|
|
||||||
if definition.minimum is not None and number < definition.minimum:
|
|
||||||
raise AppError(422, "VALUE_RANGE", f"{definition.key} is below the minimum")
|
|
||||||
if definition.maximum is not None and number > definition.maximum:
|
|
||||||
raise AppError(422, "VALUE_RANGE", f"{definition.key} is above the maximum")
|
|
||||||
if definition.canonical_unit and statement.unit != definition.canonical_unit:
|
|
||||||
raise AppError(
|
|
||||||
422,
|
|
||||||
"UNIT_MISMATCH",
|
|
||||||
"unit must match the canonical unit",
|
|
||||||
{"expected": definition.canonical_unit, "actual": statement.unit},
|
|
||||||
)
|
|
||||||
if definition.canonical_unit is None and statement.unit not in {None, ""}:
|
|
||||||
raise AppError(422, "UNIT_MISMATCH", "this attribute has no canonical unit")
|
|
||||||
return statement
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_type(definition: AttributeDefinition, value: Any) -> None:
|
|
||||||
if definition.value_type == ValueType.NUMBER:
|
|
||||||
if isinstance(value, bool) or not isinstance(value, (int, float, Decimal)):
|
|
||||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a number")
|
|
||||||
return
|
|
||||||
if definition.value_type == ValueType.STRING:
|
|
||||||
if not isinstance(value, str):
|
|
||||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a string")
|
|
||||||
return
|
|
||||||
if definition.value_type == ValueType.BOOLEAN:
|
|
||||||
if not isinstance(value, bool):
|
|
||||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a boolean")
|
|
||||||
return
|
|
||||||
if definition.value_type == ValueType.STRING_LIST:
|
|
||||||
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
|
|
||||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a list of strings")
|
|
||||||
|
|
@ -1,324 +0,0 @@
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from sqlalchemy import Boolean, DateTime, ForeignKey, ForeignKeyConstraint, Integer, Numeric, String, Text, UniqueConstraint
|
|
||||||
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from app.persistence.models import Base
|
|
||||||
|
|
||||||
|
|
||||||
class ProductCategoryRecord(Base):
|
|
||||||
__tablename__ = "product_categories"
|
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
||||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="ACTIVE")
|
|
||||||
|
|
||||||
|
|
||||||
class BuyingProjectRecord(Base):
|
|
||||||
__tablename__ = "buying_projects"
|
|
||||||
__table_args__ = (
|
|
||||||
ForeignKeyConstraint(
|
|
||||||
["template_id", "template_version"],
|
|
||||||
["product_profiles.id", "product_profiles.version"],
|
|
||||||
name="fk_buying_projects_template",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
workspace_id: Mapped[str | None] = mapped_column(String(64))
|
|
||||||
product_category_id: Mapped[str] = mapped_column(ForeignKey("product_categories.id"), nullable=False)
|
|
||||||
market: Mapped[str] = mapped_column(String(8), nullable=False)
|
|
||||||
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
|
||||||
target_budget: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
|
||||||
template_id: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
template_version: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
shortlist_size: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
|
|
||||||
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
archived_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
|
|
||||||
@property
|
|
||||||
def profile_id(self) -> str | None:
|
|
||||||
return self.template_id
|
|
||||||
|
|
||||||
@property
|
|
||||||
def profile_version(self) -> int | None:
|
|
||||||
return self.template_version
|
|
||||||
|
|
||||||
|
|
||||||
class BuyingProjectCriteriaRecord(Base):
|
|
||||||
__tablename__ = "buying_project_criteria"
|
|
||||||
|
|
||||||
project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), primary_key=True)
|
|
||||||
version: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
||||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class ProjectCriterionRecord(Base):
|
|
||||||
__tablename__ = "project_criteria"
|
|
||||||
__table_args__ = (
|
|
||||||
ForeignKeyConstraint(
|
|
||||||
["project_id", "version"],
|
|
||||||
["buying_project_criteria.project_id", "buying_project_criteria.version"],
|
|
||||||
name="fk_project_criteria_version",
|
|
||||||
),
|
|
||||||
UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
project_id: Mapped[UUID] = mapped_column(nullable=False)
|
|
||||||
version: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
item_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
|
||||||
criterion_key: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
operator: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
||||||
value: Mapped[Any] = mapped_column(JSONB)
|
|
||||||
enabled_for_discovery: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def mandatory(self) -> bool:
|
|
||||||
return self.enabled_for_discovery
|
|
||||||
|
|
||||||
@property
|
|
||||||
def criteria_version(self) -> int:
|
|
||||||
return self.version
|
|
||||||
|
|
||||||
|
|
||||||
class SourceCategoryRecord(Base):
|
|
||||||
__tablename__ = "source_categories"
|
|
||||||
__table_args__ = (UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
external_category_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
source_url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
||||||
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
|
|
||||||
canonical_category: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
taxonomy_status: Mapped[str] = mapped_column(String(32), nullable=False, default="NOT_LEARNED")
|
|
||||||
last_taxonomy_scan_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
last_verified_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
|
|
||||||
|
|
||||||
class SourceFilterDefinitionRecord(Base):
|
|
||||||
__tablename__ = "source_filter_definitions"
|
|
||||||
__table_args__ = (UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
|
||||||
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
label: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
group_label: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
filter_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
canonical_unit: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class SourceFilterOptionRecord(Base):
|
|
||||||
__tablename__ = "source_filter_options"
|
|
||||||
__table_args__ = (UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), nullable=False)
|
|
||||||
source_value: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
|
||||||
operator: Mapped[str | None] = mapped_column(String(16))
|
|
||||||
unit: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
|
|
||||||
|
|
||||||
class SourceFilterRangeRecord(Base):
|
|
||||||
__tablename__ = "source_filter_ranges"
|
|
||||||
|
|
||||||
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), primary_key=True)
|
|
||||||
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
|
||||||
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
|
||||||
step: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
|
||||||
unit: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
|
|
||||||
|
|
||||||
class CriterionDefinitionRecord(Base):
|
|
||||||
__tablename__ = "criterion_definitions"
|
|
||||||
__table_args__ = (UniqueConstraint("key", name="uq_criterion_definition_key"),)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
key: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
origin: Mapped[str] = mapped_column(String(16), nullable=False, default="SOURCE")
|
|
||||||
source: Mapped[str | None] = mapped_column(String(64))
|
|
||||||
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
|
|
||||||
label: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
unit: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
semantic: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
category: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
description: Mapped[str | None] = mapped_column(Text)
|
|
||||||
scoring_attribute_key: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
|
|
||||||
|
|
||||||
class SourceAttributeMappingRecord(Base):
|
|
||||||
__tablename__ = "source_attribute_mappings"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
source_category_id: Mapped[UUID | None] = mapped_column(ForeignKey("source_categories.id"))
|
|
||||||
raw_name: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class CriterionMappingRecord(Base):
|
|
||||||
__tablename__ = "criterion_mappings"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
|
||||||
criterion_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
|
||||||
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
source_value: Mapped[str | None] = mapped_column(String(256))
|
|
||||||
encoding: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class DiscoverySearchRecord(Base):
|
|
||||||
__tablename__ = "discovery_searches"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
|
||||||
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
schedule_interval_minutes: Mapped[int] = mapped_column(Integer, nullable=False, default=180)
|
|
||||||
broad_result_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=300)
|
|
||||||
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
last_successful_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class SearchRunRecord(Base):
|
|
||||||
__tablename__ = "search_runs"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
discovery_search_id: Mapped[UUID] = mapped_column(ForeignKey("discovery_searches.id"), nullable=False)
|
|
||||||
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
||||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
result_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
new_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
changed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
unchanged_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
missing_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
http_status_summary: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
|
|
||||||
error_code: Mapped[str | None] = mapped_column(String(64))
|
|
||||||
|
|
||||||
|
|
||||||
class SourceProductReferenceRecord(Base):
|
|
||||||
__tablename__ = "source_product_references"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
|
||||||
source_external_id: Mapped[str | None] = mapped_column(String(256))
|
|
||||||
source_url: Mapped[str | None] = mapped_column(Text)
|
|
||||||
source_title: Mapped[str] = mapped_column(String(512), nullable=False)
|
|
||||||
manufacturer: Mapped[str | None] = mapped_column(String(256))
|
|
||||||
model: Mapped[str | None] = mapped_column(String(256))
|
|
||||||
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
gtin_ean: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
normalized_spec_fingerprint: Mapped[str | None] = mapped_column(Text)
|
|
||||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
lifecycle_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
consecutive_missing_success_runs: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
|
|
||||||
|
|
||||||
|
|
||||||
class ProductObservationRecord(Base):
|
|
||||||
__tablename__ = "product_observations"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
|
||||||
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
|
|
||||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
raw_title: Mapped[str] = mapped_column(String(512), nullable=False)
|
|
||||||
best_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
|
||||||
currency: Mapped[str | None] = mapped_column(String(3))
|
|
||||||
availability: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
seller_count: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
|
|
||||||
|
|
||||||
class ProductObservationAttributeRecord(Base):
|
|
||||||
__tablename__ = "product_observation_attributes"
|
|
||||||
__table_args__ = (UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
observation_id: Mapped[UUID] = mapped_column(ForeignKey("product_observations.id"), nullable=False)
|
|
||||||
criterion_definition_id: Mapped[UUID | None] = mapped_column(ForeignKey("criterion_definitions.id"))
|
|
||||||
attribute_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
raw_name: Mapped[str | None] = mapped_column(String(256))
|
|
||||||
raw_value: Mapped[Any] = mapped_column(JSONB)
|
|
||||||
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
|
||||||
unit: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
|
|
||||||
|
|
||||||
class SearchRunItemRecord(Base):
|
|
||||||
__tablename__ = "search_run_items"
|
|
||||||
__table_args__ = (UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
|
|
||||||
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
|
||||||
observation_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_observations.id"))
|
|
||||||
diff_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class ProjectCandidateRecord(Base):
|
|
||||||
__tablename__ = "project_candidates"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
|
|
||||||
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
|
||||||
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
product_eligibility: Mapped[str | None] = mapped_column(String(16))
|
|
||||||
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class CandidateStatusHistoryRecord(Base):
|
|
||||||
__tablename__ = "candidate_status_history"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
candidate_id: Mapped[UUID] = mapped_column(ForeignKey("project_candidates.id"), nullable=False)
|
|
||||||
from_status: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
to_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
reason: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
@ -1,105 +0,0 @@
|
||||||
from decimal import Decimal
|
|
||||||
|
|
||||||
from app.domain.enums import (
|
|
||||||
Availability,
|
|
||||||
NormalizationStatus,
|
|
||||||
OfferCondition,
|
|
||||||
ValueType,
|
|
||||||
)
|
|
||||||
from app.domain.models import (
|
|
||||||
AttributeDefinition,
|
|
||||||
AttributeStatement,
|
|
||||||
Offer,
|
|
||||||
PriceObservation,
|
|
||||||
Product,
|
|
||||||
ProductVariant,
|
|
||||||
)
|
|
||||||
from app.persistence import models as orm
|
|
||||||
|
|
||||||
|
|
||||||
def decimal_or_none(value) -> Decimal | None:
|
|
||||||
return None if value is None else Decimal(str(value))
|
|
||||||
|
|
||||||
|
|
||||||
def to_definition(row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
|
|
||||||
return AttributeDefinition(
|
|
||||||
key=row.key,
|
|
||||||
value_type=ValueType(row.value_type),
|
|
||||||
canonical_unit=row.canonical_unit,
|
|
||||||
minimum=decimal_or_none(row.minimum),
|
|
||||||
maximum=decimal_or_none(row.maximum),
|
|
||||||
description=row.description,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def to_product(row: orm.ProductRecord) -> Product:
|
|
||||||
return Product(
|
|
||||||
id=row.id,
|
|
||||||
category=row.category,
|
|
||||||
brand=row.brand,
|
|
||||||
name=row.name,
|
|
||||||
model=row.model,
|
|
||||||
created_at=row.created_at,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def to_variant(row: orm.ProductVariantRecord) -> ProductVariant:
|
|
||||||
return ProductVariant(
|
|
||||||
id=row.id,
|
|
||||||
product_id=row.product_id,
|
|
||||||
name=row.name,
|
|
||||||
manufacturer_sku=row.manufacturer_sku,
|
|
||||||
identifiers=row.identifiers or {},
|
|
||||||
created_at=row.created_at,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def to_attribute(row: orm.VariantAttributeRecord) -> AttributeStatement:
|
|
||||||
return AttributeStatement(
|
|
||||||
id=row.id,
|
|
||||||
variant_id=row.variant_id,
|
|
||||||
attribute=row.attribute_key,
|
|
||||||
raw_value=row.raw_value,
|
|
||||||
normalized_value=row.normalized_value,
|
|
||||||
unit=row.unit,
|
|
||||||
normalization_status=NormalizationStatus(row.normalization_status),
|
|
||||||
normalizer_version=row.normalizer_version,
|
|
||||||
confidence=Decimal(str(row.confidence)),
|
|
||||||
source_id=row.source_id,
|
|
||||||
source_reference=row.source_reference,
|
|
||||||
observed_at=row.observed_at,
|
|
||||||
ingested_at=row.ingested_at,
|
|
||||||
normalization_error=row.normalization_error,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def to_offer(row: orm.OfferRecord) -> Offer:
|
|
||||||
return Offer(
|
|
||||||
id=row.id,
|
|
||||||
variant_id=row.variant_id,
|
|
||||||
source_id=row.source_id,
|
|
||||||
external_id=row.external_id,
|
|
||||||
seller=row.seller,
|
|
||||||
url=row.url,
|
|
||||||
market=row.market,
|
|
||||||
currency=row.currency,
|
|
||||||
condition=OfferCondition(row.condition),
|
|
||||||
created_at=row.created_at,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def to_observation(row: orm.PriceObservationRecord) -> PriceObservation:
|
|
||||||
return PriceObservation(
|
|
||||||
id=row.id,
|
|
||||||
offer_id=row.offer_id,
|
|
||||||
source_id=row.source_id,
|
|
||||||
external_observation_id=row.external_observation_id,
|
|
||||||
observed_at=row.observed_at,
|
|
||||||
ingested_at=row.ingested_at,
|
|
||||||
item_price=Decimal(str(row.item_price)),
|
|
||||||
shipping_price=decimal_or_none(row.shipping_price),
|
|
||||||
currency=row.currency,
|
|
||||||
availability=Availability(row.availability),
|
|
||||||
raw_payload=row.raw_payload,
|
|
||||||
confidence=Decimal(str(row.confidence)),
|
|
||||||
)
|
|
||||||
|
|
@ -1,249 +0,0 @@
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from sqlalchemy import (
|
|
||||||
CheckConstraint,
|
|
||||||
DateTime,
|
|
||||||
ForeignKey,
|
|
||||||
ForeignKeyConstraint,
|
|
||||||
Numeric,
|
|
||||||
String,
|
|
||||||
Text,
|
|
||||||
UniqueConstraint,
|
|
||||||
)
|
|
||||||
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
|
||||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
|
||||||
|
|
||||||
|
|
||||||
class Base(DeclarativeBase):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class AttributeDefinitionRecord(Base):
|
|
||||||
__tablename__ = "attribute_definitions"
|
|
||||||
__table_args__ = (
|
|
||||||
CheckConstraint(
|
|
||||||
"value_type IN ('number', 'string', 'boolean', 'string_list')",
|
|
||||||
name="ck_attribute_definitions_value_type",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
||||||
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
canonical_unit: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
|
|
||||||
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
|
|
||||||
description: Mapped[str | None] = mapped_column(Text)
|
|
||||||
|
|
||||||
|
|
||||||
class SourceRecord(Base):
|
|
||||||
__tablename__ = "sources"
|
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
||||||
name: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
adapter_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class ScoringModelRecord(Base):
|
|
||||||
__tablename__ = "scoring_models"
|
|
||||||
|
|
||||||
version: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
||||||
algorithm_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
|
||||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class ProductProfileRecord(Base):
|
|
||||||
__tablename__ = "product_profiles"
|
|
||||||
__table_args__ = (
|
|
||||||
CheckConstraint("version > 0", name="ck_product_profiles_version"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
||||||
version: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
category: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
|
|
||||||
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
|
||||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class ProductRecord(Base):
|
|
||||||
__tablename__ = "products"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
category: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
brand: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
model: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
variants: Mapped[list["ProductVariantRecord"]] = relationship(back_populates="product")
|
|
||||||
|
|
||||||
|
|
||||||
class ProductVariantRecord(Base):
|
|
||||||
__tablename__ = "product_variants"
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
product_id: Mapped[UUID] = mapped_column(ForeignKey("products.id"), nullable=False, index=True)
|
|
||||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
|
|
||||||
identifiers: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
product: Mapped[ProductRecord] = relationship(back_populates="variants")
|
|
||||||
|
|
||||||
|
|
||||||
class VariantAttributeRecord(Base):
|
|
||||||
__tablename__ = "variant_attributes"
|
|
||||||
__table_args__ = (
|
|
||||||
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
|
|
||||||
CheckConstraint(
|
|
||||||
"normalization_status IN ('normalized', 'unknown')",
|
|
||||||
name="ck_variant_attributes_status",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False)
|
|
||||||
attribute_key: Mapped[str] = mapped_column(ForeignKey("attribute_definitions.key"), nullable=False)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
raw_value: Mapped[Any] = mapped_column(JSONB, nullable=False)
|
|
||||||
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
|
||||||
unit: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
normalization_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
normalizer_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
|
|
||||||
source_reference: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
normalization_error: Mapped[str | None] = mapped_column(Text)
|
|
||||||
|
|
||||||
|
|
||||||
class OfferRecord(Base):
|
|
||||||
__tablename__ = "offers"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
|
|
||||||
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
|
|
||||||
CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
external_id: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
seller: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
url: Mapped[str | None] = mapped_column(Text)
|
|
||||||
market: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
||||||
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
|
||||||
condition: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class PriceObservationRecord(Base):
|
|
||||||
__tablename__ = "price_observations"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
|
|
||||||
CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
|
|
||||||
CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
|
|
||||||
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
|
|
||||||
CheckConstraint(
|
|
||||||
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
|
|
||||||
name="ck_price_obs_availability",
|
|
||||||
),
|
|
||||||
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
|
|
||||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
|
||||||
external_observation_id: Mapped[str] = mapped_column(String(256), nullable=False)
|
|
||||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
item_price: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
|
|
||||||
shipping_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
|
||||||
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
|
||||||
availability: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
raw_payload: Mapped[Any] = mapped_column(JSONB, nullable=False)
|
|
||||||
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class EvaluationRecord(Base):
|
|
||||||
__tablename__ = "evaluations"
|
|
||||||
__table_args__ = (
|
|
||||||
ForeignKeyConstraint(
|
|
||||||
["profile_id", "profile_version"],
|
|
||||||
["product_profiles.id", "product_profiles.version"],
|
|
||||||
name="fk_evaluations_profile",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
|
||||||
name="ck_eval_product_eligibility",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
|
||||||
name="ck_eval_offer_eligibility",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
|
||||||
name="ck_eval_overall_eligibility",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
|
|
||||||
name="ck_eval_product_score",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
|
|
||||||
name="ck_eval_price_score",
|
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
|
|
||||||
name="ck_eval_deal_score",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
|
|
||||||
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
|
|
||||||
profile_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
profile_version: Mapped[int] = mapped_column(nullable=False)
|
|
||||||
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
|
|
||||||
as_of: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
snapshot_cutoff: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
scoring_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
engine_build: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
product_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
||||||
offer_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
||||||
overall_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
||||||
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
|
||||||
price_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
|
||||||
deal_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
|
||||||
classification: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
price_reference_scope: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
input_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
|
||||||
result_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
components: Mapped[list["EvaluationComponentRecord"]] = relationship(back_populates="evaluation")
|
|
||||||
|
|
||||||
|
|
||||||
class EvaluationComponentRecord(Base):
|
|
||||||
__tablename__ = "evaluation_components"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
|
||||||
evaluation_id: Mapped[UUID] = mapped_column(ForeignKey("evaluations.id"), nullable=False)
|
|
||||||
kind: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
component_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
||||||
raw_score: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
|
||||||
weight: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
|
||||||
contribution: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
|
||||||
details: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
|
||||||
|
|
||||||
evaluation: Mapped[EvaluationRecord] = relationship(back_populates="components")
|
|
||||||
|
|
@ -1,183 +0,0 @@
|
||||||
from datetime import datetime
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
|
|
||||||
from app.domain.validation import parse_profile, parse_scoring_model
|
|
||||||
from app.persistence import models as orm
|
|
||||||
from app.persistence.mapping import to_attribute, to_definition, to_observation, to_offer, to_product, to_variant
|
|
||||||
|
|
||||||
|
|
||||||
class CatalogRepository:
|
|
||||||
def __init__(self, session: Session) -> None:
|
|
||||||
self.session = session
|
|
||||||
|
|
||||||
def get_definition(self, key: str) -> AttributeDefinition | None:
|
|
||||||
row = self.session.get(orm.AttributeDefinitionRecord, key)
|
|
||||||
return None if row is None else to_definition(row)
|
|
||||||
|
|
||||||
def list_definitions(self, limit: int, offset: int) -> tuple[list[AttributeDefinition], int]:
|
|
||||||
total = self.session.scalar(select(func.count()).select_from(orm.AttributeDefinitionRecord)) or 0
|
|
||||||
rows = self.session.scalars(
|
|
||||||
select(orm.AttributeDefinitionRecord).order_by(orm.AttributeDefinitionRecord.key).limit(limit).offset(offset)
|
|
||||||
).all()
|
|
||||||
return [to_definition(row) for row in rows], total
|
|
||||||
|
|
||||||
def add_definition(self, row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return to_definition(row)
|
|
||||||
|
|
||||||
def list_sources(self) -> list[orm.SourceRecord]:
|
|
||||||
return list(self.session.scalars(select(orm.SourceRecord).order_by(orm.SourceRecord.id)).all())
|
|
||||||
|
|
||||||
def get_source(self, source_id: str) -> orm.SourceRecord | None:
|
|
||||||
return self.session.get(orm.SourceRecord, source_id)
|
|
||||||
|
|
||||||
def get_scoring_model(self, version: str):
|
|
||||||
row = self.session.get(orm.ScoringModelRecord, version)
|
|
||||||
return None if row is None else parse_scoring_model(row.definition)
|
|
||||||
|
|
||||||
def scoring_versions(self) -> set[str]:
|
|
||||||
return set(self.session.scalars(select(orm.ScoringModelRecord.version)).all())
|
|
||||||
|
|
||||||
def all_definitions(self) -> dict[str, AttributeDefinition]:
|
|
||||||
rows = self.session.scalars(select(orm.AttributeDefinitionRecord)).all()
|
|
||||||
return {row.key: to_definition(row) for row in rows}
|
|
||||||
|
|
||||||
def get_profile(self, profile_id: str, version: int):
|
|
||||||
row = self.session.get(orm.ProductProfileRecord, (profile_id, version))
|
|
||||||
if row is None:
|
|
||||||
return None
|
|
||||||
model = self.get_scoring_model(row.scoring_version)
|
|
||||||
if model is None:
|
|
||||||
return None
|
|
||||||
return parse_profile(
|
|
||||||
row.definition,
|
|
||||||
attribute_definitions=self.all_definitions(),
|
|
||||||
scoring_versions=self.scoring_versions(),
|
|
||||||
supported_strategies=set(model.supported_preference_strategies),
|
|
||||||
)
|
|
||||||
|
|
||||||
def add_profile(self, row: orm.ProductProfileRecord):
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return row
|
|
||||||
|
|
||||||
def list_profiles(self, limit: int, offset: int) -> tuple[list[orm.ProductProfileRecord], int]:
|
|
||||||
total = self.session.scalar(select(func.count()).select_from(orm.ProductProfileRecord)) or 0
|
|
||||||
rows = self.session.scalars(
|
|
||||||
select(orm.ProductProfileRecord)
|
|
||||||
.order_by(orm.ProductProfileRecord.id, orm.ProductProfileRecord.version)
|
|
||||||
.limit(limit)
|
|
||||||
.offset(offset)
|
|
||||||
).all()
|
|
||||||
return list(rows), total
|
|
||||||
|
|
||||||
def get_product(self, product_id: UUID) -> Product | None:
|
|
||||||
row = self.session.get(orm.ProductRecord, product_id)
|
|
||||||
return None if row is None else to_product(row)
|
|
||||||
|
|
||||||
def add_product(self, row: orm.ProductRecord) -> Product:
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return to_product(row)
|
|
||||||
|
|
||||||
def get_variant(self, variant_id: UUID) -> ProductVariant | None:
|
|
||||||
row = self.session.get(orm.ProductVariantRecord, variant_id)
|
|
||||||
return None if row is None else to_variant(row)
|
|
||||||
|
|
||||||
def add_variant(self, row: orm.ProductVariantRecord) -> ProductVariant:
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return to_variant(row)
|
|
||||||
|
|
||||||
def list_attributes(self, variant_id: UUID) -> list[AttributeStatement]:
|
|
||||||
rows = self.session.scalars(
|
|
||||||
select(orm.VariantAttributeRecord)
|
|
||||||
.where(orm.VariantAttributeRecord.variant_id == variant_id)
|
|
||||||
.order_by(
|
|
||||||
orm.VariantAttributeRecord.observed_at,
|
|
||||||
orm.VariantAttributeRecord.ingested_at,
|
|
||||||
orm.VariantAttributeRecord.id,
|
|
||||||
)
|
|
||||||
).all()
|
|
||||||
return [to_attribute(row) for row in rows]
|
|
||||||
|
|
||||||
def get_attribute(self, attribute_id: UUID) -> AttributeStatement | None:
|
|
||||||
row = self.session.get(orm.VariantAttributeRecord, attribute_id)
|
|
||||||
return None if row is None else to_attribute(row)
|
|
||||||
|
|
||||||
def add_attribute(self, row: orm.VariantAttributeRecord) -> AttributeStatement:
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return to_attribute(row)
|
|
||||||
|
|
||||||
def get_offer(self, offer_id: UUID) -> Offer | None:
|
|
||||||
row = self.session.get(orm.OfferRecord, offer_id)
|
|
||||||
return None if row is None else to_offer(row)
|
|
||||||
|
|
||||||
def get_offer_by_external(self, source_id: str, external_id: str) -> Offer | None:
|
|
||||||
row = self.session.scalar(
|
|
||||||
select(orm.OfferRecord).where(
|
|
||||||
orm.OfferRecord.source_id == source_id,
|
|
||||||
orm.OfferRecord.external_id == external_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return None if row is None else to_offer(row)
|
|
||||||
|
|
||||||
def add_offer(self, row: orm.OfferRecord) -> Offer:
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return to_offer(row)
|
|
||||||
|
|
||||||
def list_observations(self, offer_id: UUID, limit: int | None = None, offset: int = 0) -> tuple[list[PriceObservation], int]:
|
|
||||||
total = (
|
|
||||||
self.session.scalar(
|
|
||||||
select(func.count())
|
|
||||||
.select_from(orm.PriceObservationRecord)
|
|
||||||
.where(orm.PriceObservationRecord.offer_id == offer_id)
|
|
||||||
)
|
|
||||||
or 0
|
|
||||||
)
|
|
||||||
query = (
|
|
||||||
select(orm.PriceObservationRecord)
|
|
||||||
.where(orm.PriceObservationRecord.offer_id == offer_id)
|
|
||||||
.order_by(
|
|
||||||
orm.PriceObservationRecord.observed_at,
|
|
||||||
orm.PriceObservationRecord.ingested_at,
|
|
||||||
orm.PriceObservationRecord.id,
|
|
||||||
)
|
|
||||||
.offset(offset)
|
|
||||||
)
|
|
||||||
if limit is not None:
|
|
||||||
query = query.limit(limit)
|
|
||||||
rows = self.session.scalars(query).all()
|
|
||||||
return [to_observation(row) for row in rows], total
|
|
||||||
|
|
||||||
def get_observation_by_external(self, offer_id: UUID, external_id: str) -> orm.PriceObservationRecord | None:
|
|
||||||
return self.session.scalar(
|
|
||||||
select(orm.PriceObservationRecord).where(
|
|
||||||
orm.PriceObservationRecord.offer_id == offer_id,
|
|
||||||
orm.PriceObservationRecord.external_observation_id == external_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def add_observation(self, row: orm.PriceObservationRecord) -> PriceObservation:
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return to_observation(row)
|
|
||||||
|
|
||||||
def get_evaluation(self, evaluation_id: UUID) -> orm.EvaluationRecord | None:
|
|
||||||
return self.session.get(orm.EvaluationRecord, evaluation_id)
|
|
||||||
|
|
||||||
def add_evaluation(self, row: orm.EvaluationRecord) -> orm.EvaluationRecord:
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
return row
|
|
||||||
|
|
||||||
def add_component(self, row: orm.EvaluationComponentRecord) -> None:
|
|
||||||
self.session.add(row)
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
from decimal import Decimal
|
|
||||||
|
|
||||||
from app.domain.enums import Classification, Eligibility
|
|
||||||
from app.domain.models import ScoringModel
|
|
||||||
from app.scoring.mathutil import clamp
|
|
||||||
|
|
||||||
|
|
||||||
def classify_deal_score(score: Decimal, model: ScoringModel) -> str:
|
|
||||||
for bound in model.classification_upper_bounds:
|
|
||||||
if bound.inclusive and score <= bound.upper:
|
|
||||||
return bound.label
|
|
||||||
if not bound.inclusive and score < bound.upper:
|
|
||||||
return bound.label
|
|
||||||
return Classification.EXCEPTIONAL.value
|
|
||||||
|
|
||||||
|
|
||||||
def score_deal(product_score: Decimal, price_score: Decimal, model: ScoringModel) -> Decimal:
|
|
||||||
return clamp(model.product_weight * product_score + model.price_weight * price_score)
|
|
||||||
|
|
||||||
|
|
||||||
def eligibility_from_parts(product: Eligibility, offer: Eligibility) -> Eligibility:
|
|
||||||
if Eligibility.INELIGIBLE in {product, offer}:
|
|
||||||
return Eligibility.INELIGIBLE
|
|
||||||
if Eligibility.UNKNOWN in {product, offer}:
|
|
||||||
return Eligibility.UNKNOWN
|
|
||||||
return Eligibility.ELIGIBLE
|
|
||||||
|
|
||||||
|
|
||||||
def derive_eligible(overall: Eligibility) -> bool | None:
|
|
||||||
if overall == Eligibility.ELIGIBLE:
|
|
||||||
return True
|
|
||||||
if overall == Eligibility.INELIGIBLE:
|
|
||||||
return False
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def classification_for(overall: Eligibility, deal_score: Decimal | None, model: ScoringModel) -> str:
|
|
||||||
if overall == Eligibility.INELIGIBLE:
|
|
||||||
return Classification.INELIGIBLE.value
|
|
||||||
if overall == Eligibility.UNKNOWN or deal_score is None:
|
|
||||||
return Classification.INSUFFICIENT_DATA.value
|
|
||||||
return classify_deal_score(deal_score, model)
|
|
||||||
|
|
@ -1,545 +0,0 @@
|
||||||
from collections.abc import Sequence
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from app.domain.attributes import select_attribute_snapshot
|
|
||||||
from app.domain.constraints import evaluate_constraint
|
|
||||||
from app.domain.enums import (
|
|
||||||
Availability,
|
|
||||||
ConstraintStatus,
|
|
||||||
Eligibility,
|
|
||||||
PriceReferenceScope,
|
|
||||||
)
|
|
||||||
from app.domain.models import (
|
|
||||||
AttributeDefinition,
|
|
||||||
AttributeStatement,
|
|
||||||
ConstraintResult,
|
|
||||||
EvaluationResult,
|
|
||||||
GateResult,
|
|
||||||
Offer,
|
|
||||||
PriceObservation,
|
|
||||||
Product,
|
|
||||||
ProductVariant,
|
|
||||||
Profile,
|
|
||||||
Reason,
|
|
||||||
ScoreComponent,
|
|
||||||
ScoringModel,
|
|
||||||
)
|
|
||||||
from app.domain.serialize import jsonable
|
|
||||||
from app.scoring.deal import classification_for, derive_eligible, eligibility_from_parts, score_deal
|
|
||||||
from app.scoring.mathutil import round_score
|
|
||||||
from app.scoring.price import (
|
|
||||||
build_offer_reference_history,
|
|
||||||
price_statistics_from_windows,
|
|
||||||
score_price,
|
|
||||||
select_current_observation,
|
|
||||||
)
|
|
||||||
from app.scoring.product import score_product
|
|
||||||
|
|
||||||
|
|
||||||
def evaluate_deal(
|
|
||||||
*,
|
|
||||||
profile: Profile,
|
|
||||||
scoring_model: ScoringModel,
|
|
||||||
product: Product,
|
|
||||||
variant: ProductVariant,
|
|
||||||
offer: Offer,
|
|
||||||
attributes: tuple[AttributeStatement, ...],
|
|
||||||
observations: tuple[PriceObservation, ...],
|
|
||||||
definitions: dict[str, AttributeDefinition],
|
|
||||||
as_of: datetime,
|
|
||||||
snapshot_cutoff: datetime,
|
|
||||||
engine_build: str,
|
|
||||||
) -> EvaluationResult:
|
|
||||||
observations = tuple(item for item in observations if item.offer_id == offer.id)
|
|
||||||
snapshot = select_attribute_snapshot(attributes, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
|
|
||||||
constraint_results = tuple(
|
|
||||||
evaluate_constraint(
|
|
||||||
constraint,
|
|
||||||
snapshot.get(constraint.attribute),
|
|
||||||
definitions[constraint.attribute],
|
|
||||||
scoring_model.minimum_confidence,
|
|
||||||
)
|
|
||||||
for constraint in profile.constraints
|
|
||||||
)
|
|
||||||
product_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
|
|
||||||
product_score_raw, coverage, preference_components = score_product(
|
|
||||||
profile.preferences,
|
|
||||||
snapshot,
|
|
||||||
constraint_results,
|
|
||||||
scoring_model,
|
|
||||||
)
|
|
||||||
|
|
||||||
current = select_current_observation(observations, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
|
|
||||||
offer_gates, offer_eligibility = _offer_eligibility(profile, offer, current, scoring_model, as_of)
|
|
||||||
overall = eligibility_from_parts(product_eligibility, offer_eligibility)
|
|
||||||
|
|
||||||
windows: dict[int, Any] = {}
|
|
||||||
price_score_raw: Decimal | None = None
|
|
||||||
budget_score: Decimal | None = None
|
|
||||||
history_score: Decimal | None = None
|
|
||||||
price_basis = None
|
|
||||||
history_warning: str | None = None
|
|
||||||
if _can_score_price(current, profile, scoring_model, as_of):
|
|
||||||
windows = build_offer_reference_history(observations, current, scoring_model)
|
|
||||||
price_score_raw, budget_score, history_score, price_basis, history_warning = score_price(
|
|
||||||
current_price=current.total_price,
|
|
||||||
reference_history=windows[90].values,
|
|
||||||
budget=profile.budget,
|
|
||||||
model=scoring_model,
|
|
||||||
)
|
|
||||||
|
|
||||||
stats = price_statistics_from_windows(current, windows)
|
|
||||||
deal_score_raw = None
|
|
||||||
if overall == Eligibility.ELIGIBLE and price_score_raw is not None:
|
|
||||||
deal_score_raw = score_deal(product_score_raw, price_score_raw, scoring_model)
|
|
||||||
|
|
||||||
classification = classification_for(overall, deal_score_raw, scoring_model)
|
|
||||||
reasons, warnings = _reasons(
|
|
||||||
constraint_results,
|
|
||||||
product_gates,
|
|
||||||
offer_gates,
|
|
||||||
stats,
|
|
||||||
current,
|
|
||||||
history_warning,
|
|
||||||
price_score_raw,
|
|
||||||
)
|
|
||||||
components = _all_components(
|
|
||||||
preference_components,
|
|
||||||
product_gates,
|
|
||||||
offer_gates,
|
|
||||||
budget_score,
|
|
||||||
history_score,
|
|
||||||
scoring_model,
|
|
||||||
)
|
|
||||||
input_snapshot = {
|
|
||||||
"profile": profile.definition,
|
|
||||||
"scoring_model": scoring_model.definition,
|
|
||||||
"product": jsonable(_product_dict(product)),
|
|
||||||
"variant": jsonable(_variant_dict(variant)),
|
|
||||||
"offer": jsonable(_offer_dict(offer)),
|
|
||||||
"attribute_snapshot": {
|
|
||||||
key: jsonable(_attribute_dict(value))
|
|
||||||
for key, value in snapshot.items()
|
|
||||||
},
|
|
||||||
"current_observation": None if current is None else jsonable(_observation_dict(current)),
|
|
||||||
"history_observation_ids": [str(item) for item in stats.used_observation_ids],
|
|
||||||
"as_of": jsonable(as_of),
|
|
||||||
"snapshot_cutoff": jsonable(snapshot_cutoff),
|
|
||||||
"price_reference_scope": PriceReferenceScope.OFFER.value,
|
|
||||||
"unrounded": {
|
|
||||||
"product_score": jsonable(product_score_raw),
|
|
||||||
"price_score": jsonable(price_score_raw),
|
|
||||||
"deal_score": jsonable(deal_score_raw),
|
|
||||||
"budget_score": jsonable(budget_score),
|
|
||||||
"history_score": jsonable(history_score),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return EvaluationResult(
|
|
||||||
profile_id=profile.id,
|
|
||||||
profile_version=profile.version,
|
|
||||||
scoring_version=scoring_model.version,
|
|
||||||
scoring_hash=scoring_model.definition_hash,
|
|
||||||
engine_build=engine_build,
|
|
||||||
variant_id=variant.id,
|
|
||||||
offer_id=offer.id,
|
|
||||||
as_of=as_of,
|
|
||||||
snapshot_cutoff=snapshot_cutoff,
|
|
||||||
product_eligibility=product_eligibility,
|
|
||||||
offer_eligibility=offer_eligibility,
|
|
||||||
overall_eligibility=overall,
|
|
||||||
eligible=derive_eligible(overall),
|
|
||||||
product_score=round_score(product_score_raw),
|
|
||||||
product_score_raw=product_score_raw,
|
|
||||||
price_score=None if price_score_raw is None else round_score(price_score_raw),
|
|
||||||
price_score_raw=price_score_raw,
|
|
||||||
deal_score=None if deal_score_raw is None else round_score(deal_score_raw),
|
|
||||||
deal_score_raw=deal_score_raw,
|
|
||||||
classification=classification,
|
|
||||||
price_basis=price_basis,
|
|
||||||
product_coverage=coverage,
|
|
||||||
constraint_results=constraint_results,
|
|
||||||
product_gates=product_gates,
|
|
||||||
offer_gates=offer_gates,
|
|
||||||
components=components,
|
|
||||||
price_statistics=stats,
|
|
||||||
price_reference_scope=PriceReferenceScope.OFFER,
|
|
||||||
reasons=reasons,
|
|
||||||
warnings=warnings,
|
|
||||||
input_snapshot=input_snapshot,
|
|
||||||
failed_constraints=tuple(
|
|
||||||
item for item in constraint_results if item.status == ConstraintStatus.FAIL.value
|
|
||||||
),
|
|
||||||
unknown_constraints=tuple(
|
|
||||||
item for item in constraint_results if item.status == ConstraintStatus.UNKNOWN.value
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def statement_price_confidence(observation: PriceObservation, model: ScoringModel) -> bool:
|
|
||||||
return observation.confidence >= model.minimum_confidence
|
|
||||||
|
|
||||||
|
|
||||||
def _fresh_enough(observation: PriceObservation, as_of: datetime, model: ScoringModel) -> bool:
|
|
||||||
return as_of - observation.observed_at <= timedelta(hours=model.maximum_price_age_hours)
|
|
||||||
|
|
||||||
|
|
||||||
def _product_eligibility(
|
|
||||||
profile: Profile,
|
|
||||||
product: Product,
|
|
||||||
constraint_results: tuple[ConstraintResult, ...],
|
|
||||||
) -> tuple[tuple[GateResult, ...], Eligibility]:
|
|
||||||
category_ok = product.category == profile.category
|
|
||||||
category_gate = GateResult(
|
|
||||||
key="category",
|
|
||||||
status="PASS" if category_ok else "FAIL",
|
|
||||||
reason_code="CATEGORY_MATCH" if category_ok else "CATEGORY_MISMATCH",
|
|
||||||
details={"expected": profile.category, "actual": product.category},
|
|
||||||
)
|
|
||||||
mandatory = [item for item in constraint_results if item.mandatory]
|
|
||||||
if not category_ok or any(item.status == ConstraintStatus.FAIL.value for item in mandatory):
|
|
||||||
eligibility = Eligibility.INELIGIBLE
|
|
||||||
elif any(item.status == ConstraintStatus.UNKNOWN.value for item in mandatory):
|
|
||||||
eligibility = Eligibility.UNKNOWN
|
|
||||||
else:
|
|
||||||
eligibility = Eligibility.ELIGIBLE
|
|
||||||
return (category_gate,), eligibility
|
|
||||||
|
|
||||||
|
|
||||||
def _offer_eligibility(
|
|
||||||
profile: Profile,
|
|
||||||
offer: Offer,
|
|
||||||
current: PriceObservation | None,
|
|
||||||
model: ScoringModel,
|
|
||||||
as_of: datetime,
|
|
||||||
) -> tuple[tuple[GateResult, ...], Eligibility]:
|
|
||||||
gates: list[GateResult] = []
|
|
||||||
|
|
||||||
market_ok = offer.market == profile.market
|
|
||||||
gates.append(_gate("market", market_ok, "MARKET_MATCH", "MARKET_MISMATCH", offer.market, profile.market))
|
|
||||||
|
|
||||||
condition_ok = offer.condition in profile.allowed_conditions
|
|
||||||
gates.append(
|
|
||||||
_gate(
|
|
||||||
"condition",
|
|
||||||
condition_ok,
|
|
||||||
"CONDITION_ALLOWED",
|
|
||||||
"CONDITION_NOT_ALLOWED",
|
|
||||||
offer.condition.value,
|
|
||||||
[item.value for item in profile.allowed_conditions],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
currency_ok = offer.currency == profile.currency
|
|
||||||
gates.append(_gate("currency", currency_ok, "CURRENCY_MATCH", "CURRENCY_MISMATCH", offer.currency, profile.currency))
|
|
||||||
|
|
||||||
if current is None:
|
|
||||||
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_MISSING", {}))
|
|
||||||
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_MISSING", {}))
|
|
||||||
gates.append(GateResult("price_confidence", "UNKNOWN", "PRICE_MISSING", {}))
|
|
||||||
gates.append(GateResult("shipping", "UNKNOWN", "SHIPPING_UNKNOWN", {}))
|
|
||||||
gates.append(GateResult("budget", "UNKNOWN", "PRICE_MISSING", {}))
|
|
||||||
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
|
|
||||||
return tuple(gates), _combine_offer_gates(gates)
|
|
||||||
|
|
||||||
fresh = _fresh_enough(current, as_of, model)
|
|
||||||
if not fresh:
|
|
||||||
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_STALE", {"observed_at": current.observed_at.isoformat()}))
|
|
||||||
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_STALE", {}))
|
|
||||||
else:
|
|
||||||
gates.append(GateResult("price_present", "PASS", "PRICE_PRESENT", {"observation_id": str(current.id)}))
|
|
||||||
gates.append(GateResult("price_fresh", "PASS", "PRICE_FRESH", {}))
|
|
||||||
|
|
||||||
confident = statement_price_confidence(current, model)
|
|
||||||
gates.append(
|
|
||||||
GateResult(
|
|
||||||
"price_confidence",
|
|
||||||
"PASS" if confident else "UNKNOWN",
|
|
||||||
"PRICE_CONFIDENCE_OK" if confident else "PRICE_LOW_CONFIDENCE",
|
|
||||||
{"confidence": str(current.confidence)},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
shipping_known = current.shipping_price is not None
|
|
||||||
gates.append(
|
|
||||||
GateResult(
|
|
||||||
"shipping",
|
|
||||||
"PASS" if shipping_known else "UNKNOWN",
|
|
||||||
"SHIPPING_KNOWN" if shipping_known else "SHIPPING_UNKNOWN",
|
|
||||||
{},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
total = current.total_price
|
|
||||||
if total is None or not currency_ok:
|
|
||||||
gates.append(GateResult("budget", "UNKNOWN", "PRICE_INCOMPLETE", {}))
|
|
||||||
elif total <= profile.budget.maximum:
|
|
||||||
gates.append(GateResult("budget", "PASS", "WITHIN_BUDGET", {"total": str(total)}))
|
|
||||||
else:
|
|
||||||
gates.append(GateResult("budget", "FAIL", "OVER_BUDGET", {"total": str(total)}))
|
|
||||||
|
|
||||||
if current.availability == Availability.IN_STOCK:
|
|
||||||
gates.append(GateResult("availability", "PASS", "IN_STOCK", {}))
|
|
||||||
elif current.availability == Availability.OUT_OF_STOCK:
|
|
||||||
gates.append(GateResult("availability", "FAIL", "OUT_OF_STOCK", {}))
|
|
||||||
else:
|
|
||||||
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
|
|
||||||
|
|
||||||
if current.currency != offer.currency:
|
|
||||||
gates.append(GateResult("observation_currency", "FAIL", "CURRENCY_MISMATCH", {}))
|
|
||||||
|
|
||||||
if current.source_id != offer.source_id:
|
|
||||||
gates.append(GateResult("observation_source", "FAIL", "SOURCE_MISMATCH", {}))
|
|
||||||
|
|
||||||
return tuple(gates), _combine_offer_gates(gates)
|
|
||||||
|
|
||||||
|
|
||||||
def _gate(key: str, ok: bool, pass_code: str, fail_code: str, actual: Any, expected: Any) -> GateResult:
|
|
||||||
return GateResult(
|
|
||||||
key=key,
|
|
||||||
status="PASS" if ok else "FAIL",
|
|
||||||
reason_code=pass_code if ok else fail_code,
|
|
||||||
details={"actual": actual, "expected": expected},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _can_score_price(
|
|
||||||
current: PriceObservation | None,
|
|
||||||
profile: Profile,
|
|
||||||
model: ScoringModel,
|
|
||||||
as_of: datetime,
|
|
||||||
) -> bool:
|
|
||||||
if current is None:
|
|
||||||
return False
|
|
||||||
if current.currency != profile.currency:
|
|
||||||
return False
|
|
||||||
if current.total_price is None:
|
|
||||||
return False
|
|
||||||
if not _fresh_enough(current, as_of, model):
|
|
||||||
return False
|
|
||||||
return statement_price_confidence(current, model)
|
|
||||||
|
|
||||||
|
|
||||||
def _combine_offer_gates(gates: Sequence[GateResult]) -> Eligibility:
|
|
||||||
statuses = [gate.status for gate in gates]
|
|
||||||
if "FAIL" in statuses:
|
|
||||||
return Eligibility.INELIGIBLE
|
|
||||||
if "UNKNOWN" in statuses:
|
|
||||||
return Eligibility.UNKNOWN
|
|
||||||
return Eligibility.ELIGIBLE
|
|
||||||
|
|
||||||
|
|
||||||
def _reasons(
|
|
||||||
constraints: tuple[ConstraintResult, ...],
|
|
||||||
product_gates: tuple[GateResult, ...],
|
|
||||||
offer_gates: tuple[GateResult, ...],
|
|
||||||
stats,
|
|
||||||
current: PriceObservation | None,
|
|
||||||
history_warning: str | None,
|
|
||||||
price_score_raw: Decimal | None,
|
|
||||||
) -> tuple[tuple[Reason, ...], tuple[str, ...]]:
|
|
||||||
reasons: list[Reason] = []
|
|
||||||
warnings: list[str] = []
|
|
||||||
for gate in (*product_gates, *offer_gates):
|
|
||||||
if gate.status != "PASS":
|
|
||||||
reasons.append(Reason(gate.reason_code, _gate_message(gate)))
|
|
||||||
for item in constraints:
|
|
||||||
if item.status != ConstraintStatus.PASS.value:
|
|
||||||
reasons.append(Reason(item.reason_code, _constraint_message(item)))
|
|
||||||
if history_warning:
|
|
||||||
warnings.append(history_warning)
|
|
||||||
reasons.append(Reason(history_warning, "Zu wenige Historientage für den 90-Tage-Median."))
|
|
||||||
if stats.lowest_365d is not None and not stats.lowest_365d_is_full_year:
|
|
||||||
warnings.append("LOWEST_365D_PARTIAL")
|
|
||||||
reasons.append(
|
|
||||||
Reason(
|
|
||||||
"LOWEST_365D_PARTIAL",
|
|
||||||
f"lowest_365d {stats.lowest_365d} ist das Minimum der vorhandenen {stats.lowest_365d_days} Tage, kein belegtes Jahrestief.",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if (
|
|
||||||
price_score_raw is not None
|
|
||||||
and current is not None
|
|
||||||
and current.total_price is not None
|
|
||||||
and stats.median_90d is not None
|
|
||||||
and stats.median_90d != 0
|
|
||||||
):
|
|
||||||
delta = (stats.median_90d - current.total_price) / stats.median_90d * Decimal("100")
|
|
||||||
percent = round_score(delta)
|
|
||||||
if current.total_price < stats.median_90d:
|
|
||||||
reasons.append(
|
|
||||||
Reason(
|
|
||||||
"PRICE_BELOW_MEDIAN",
|
|
||||||
f"Gesamtpreis {_de_decimal(percent)} % unter dem Median der beobachteten 90-Tage-Historie.",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
elif current.total_price > stats.median_90d:
|
|
||||||
reasons.append(
|
|
||||||
Reason(
|
|
||||||
"PRICE_ABOVE_MEDIAN",
|
|
||||||
f"Gesamtpreis {_de_decimal(percent.copy_abs())} % über dem Median der beobachteten 90-Tage-Historie.",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
reasons.append(Reason("PRICE_AT_MEDIAN", "Gesamtpreis entspricht dem Median der 90-Tage-Historie."))
|
|
||||||
return tuple(reasons), tuple(warnings)
|
|
||||||
|
|
||||||
|
|
||||||
def _de_decimal(value: Decimal) -> str:
|
|
||||||
return format(value, "f").replace(".", ",")
|
|
||||||
|
|
||||||
|
|
||||||
def _gate_message(gate: GateResult) -> str:
|
|
||||||
messages = {
|
|
||||||
"CATEGORY_MISMATCH": "Die Produktkategorie entspricht nicht dem Profil.",
|
|
||||||
"MARKET_MISMATCH": "Der Angebotsmarkt entspricht nicht dem Profil.",
|
|
||||||
"CONDITION_NOT_ALLOWED": "Der Angebotszustand ist im Profil nicht erlaubt.",
|
|
||||||
"CURRENCY_MISMATCH": "Die Währung entspricht nicht dem Profil.",
|
|
||||||
"PRICE_MISSING": "Es liegt keine aktuelle Preisbeobachtung vor.",
|
|
||||||
"PRICE_STALE": "Die aktuelle Preisbeobachtung ist nicht frisch genug.",
|
|
||||||
"PRICE_LOW_CONFIDENCE": "Die Preis-Confidence liegt unter der Mindestschwelle.",
|
|
||||||
"SHIPPING_UNKNOWN": "Obligatorische Versandkosten sind unbekannt.",
|
|
||||||
"OVER_BUDGET": "Der Gesamtpreis liegt über dem Budgetmaximum.",
|
|
||||||
"OUT_OF_STOCK": "Das Angebot ist nicht lieferbar.",
|
|
||||||
"AVAILABILITY_UNKNOWN": "Die Verfügbarkeit ist unbekannt.",
|
|
||||||
"SOURCE_MISMATCH": "Die Beobachtungsquelle weicht vom Angebot ab.",
|
|
||||||
}
|
|
||||||
return messages.get(gate.reason_code, gate.reason_code)
|
|
||||||
|
|
||||||
|
|
||||||
def _constraint_message(item: ConstraintResult) -> str:
|
|
||||||
if item.status == ConstraintStatus.FAIL.value:
|
|
||||||
return f"Constraint {item.constraint_id} ist nicht erfüllt."
|
|
||||||
return f"Constraint {item.constraint_id} kann nicht bewertet werden."
|
|
||||||
|
|
||||||
|
|
||||||
def _all_components(
|
|
||||||
preference_components: tuple[ScoreComponent, ...],
|
|
||||||
product_gates: tuple[GateResult, ...],
|
|
||||||
offer_gates: tuple[GateResult, ...],
|
|
||||||
budget_score: Decimal | None,
|
|
||||||
history_score: Decimal | None,
|
|
||||||
model: ScoringModel,
|
|
||||||
) -> tuple[ScoreComponent, ...]:
|
|
||||||
extra: list[ScoreComponent] = list(preference_components)
|
|
||||||
if budget_score is not None:
|
|
||||||
extra.append(
|
|
||||||
ScoreComponent(
|
|
||||||
kind="budget",
|
|
||||||
component_key="budget",
|
|
||||||
raw_score=budget_score,
|
|
||||||
weight=model.budget_weight,
|
|
||||||
contribution=model.budget_weight * budget_score,
|
|
||||||
details={},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if history_score is not None:
|
|
||||||
extra.append(
|
|
||||||
ScoreComponent(
|
|
||||||
kind="history",
|
|
||||||
component_key="history",
|
|
||||||
raw_score=history_score,
|
|
||||||
weight=model.history_weight,
|
|
||||||
contribution=model.history_weight * history_score,
|
|
||||||
details={},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
for gate in product_gates:
|
|
||||||
extra.append(
|
|
||||||
ScoreComponent(
|
|
||||||
kind="product_gate",
|
|
||||||
component_key=gate.key,
|
|
||||||
raw_score=None,
|
|
||||||
weight=None,
|
|
||||||
contribution=None,
|
|
||||||
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
for gate in offer_gates:
|
|
||||||
extra.append(
|
|
||||||
ScoreComponent(
|
|
||||||
kind="offer_gate",
|
|
||||||
component_key=gate.key,
|
|
||||||
raw_score=None,
|
|
||||||
weight=None,
|
|
||||||
contribution=None,
|
|
||||||
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return tuple(extra)
|
|
||||||
|
|
||||||
|
|
||||||
def _product_dict(product: Product) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": product.id,
|
|
||||||
"category": product.category,
|
|
||||||
"brand": product.brand,
|
|
||||||
"name": product.name,
|
|
||||||
"model": product.model,
|
|
||||||
"created_at": product.created_at,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _variant_dict(variant: ProductVariant) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": variant.id,
|
|
||||||
"product_id": variant.product_id,
|
|
||||||
"name": variant.name,
|
|
||||||
"manufacturer_sku": variant.manufacturer_sku,
|
|
||||||
"identifiers": variant.identifiers,
|
|
||||||
"created_at": variant.created_at,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _offer_dict(offer: Offer) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": offer.id,
|
|
||||||
"variant_id": offer.variant_id,
|
|
||||||
"source_id": offer.source_id,
|
|
||||||
"external_id": offer.external_id,
|
|
||||||
"seller": offer.seller,
|
|
||||||
"url": offer.url,
|
|
||||||
"market": offer.market,
|
|
||||||
"currency": offer.currency,
|
|
||||||
"condition": offer.condition,
|
|
||||||
"created_at": offer.created_at,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _attribute_dict(statement: AttributeStatement) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": statement.id,
|
|
||||||
"variant_id": statement.variant_id,
|
|
||||||
"attribute": statement.attribute,
|
|
||||||
"raw_value": statement.raw_value,
|
|
||||||
"normalized_value": statement.normalized_value,
|
|
||||||
"unit": statement.unit,
|
|
||||||
"normalization_status": statement.normalization_status,
|
|
||||||
"normalizer_version": statement.normalizer_version,
|
|
||||||
"confidence": statement.confidence,
|
|
||||||
"source_id": statement.source_id,
|
|
||||||
"source_reference": statement.source_reference,
|
|
||||||
"observed_at": statement.observed_at,
|
|
||||||
"ingested_at": statement.ingested_at,
|
|
||||||
"normalization_error": statement.normalization_error,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _observation_dict(observation: PriceObservation) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": observation.id,
|
|
||||||
"offer_id": observation.offer_id,
|
|
||||||
"source_id": observation.source_id,
|
|
||||||
"external_observation_id": observation.external_observation_id,
|
|
||||||
"observed_at": observation.observed_at,
|
|
||||||
"ingested_at": observation.ingested_at,
|
|
||||||
"item_price": observation.item_price,
|
|
||||||
"shipping_price": observation.shipping_price,
|
|
||||||
"currency": observation.currency,
|
|
||||||
"availability": observation.availability,
|
|
||||||
"raw_payload": observation.raw_payload,
|
|
||||||
"confidence": observation.confidence,
|
|
||||||
"total_price": observation.total_price,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
from decimal import ROUND_HALF_UP, Decimal
|
|
||||||
|
|
||||||
HUNDRED = Decimal("100")
|
|
||||||
ZERO = Decimal("0")
|
|
||||||
TWO_PLACES = Decimal("0.01")
|
|
||||||
|
|
||||||
|
|
||||||
def clamp(value: Decimal) -> Decimal:
|
|
||||||
return min(HUNDRED, max(ZERO, value))
|
|
||||||
|
|
||||||
|
|
||||||
def round_score(value: Decimal) -> Decimal:
|
|
||||||
return value.quantize(TWO_PLACES, rounding=ROUND_HALF_UP)
|
|
||||||
|
|
||||||
|
|
||||||
def as_decimal(value: object) -> Decimal:
|
|
||||||
if isinstance(value, Decimal):
|
|
||||||
return value
|
|
||||||
if isinstance(value, bool) or value is None:
|
|
||||||
raise TypeError("value is not numeric")
|
|
||||||
return Decimal(str(value))
|
|
||||||
|
|
@ -1,148 +0,0 @@
|
||||||
from collections.abc import Sequence
|
|
||||||
from datetime import date, datetime, timedelta
|
|
||||||
from decimal import Decimal
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from app.domain.enums import Availability, PriceBasis
|
|
||||||
from app.domain.models import Budget, HistoryWindow, PriceObservation, PriceStatistics, ScoringModel
|
|
||||||
from app.scoring.mathutil import clamp
|
|
||||||
|
|
||||||
|
|
||||||
def total_or_none(observation: PriceObservation) -> Decimal | None:
|
|
||||||
return observation.total_price
|
|
||||||
|
|
||||||
|
|
||||||
def select_current_observation(
|
|
||||||
observations: Sequence[PriceObservation],
|
|
||||||
*,
|
|
||||||
as_of: datetime,
|
|
||||||
snapshot_cutoff: datetime,
|
|
||||||
) -> PriceObservation | None:
|
|
||||||
eligible = [
|
|
||||||
item
|
|
||||||
for item in observations
|
|
||||||
if item.observed_at <= as_of and item.ingested_at <= snapshot_cutoff
|
|
||||||
]
|
|
||||||
if not eligible:
|
|
||||||
return None
|
|
||||||
return max(eligible, key=lambda item: (item.observed_at, item.ingested_at, item.id))
|
|
||||||
|
|
||||||
|
|
||||||
def median(values: Sequence[Decimal]) -> Decimal | None:
|
|
||||||
if not values:
|
|
||||||
return None
|
|
||||||
ordered = sorted(values)
|
|
||||||
mid = len(ordered) // 2
|
|
||||||
if len(ordered) % 2:
|
|
||||||
return ordered[mid]
|
|
||||||
return (ordered[mid - 1] + ordered[mid]) / Decimal("2")
|
|
||||||
|
|
||||||
|
|
||||||
def _quality_ok(observation: PriceObservation, minimum_confidence: Decimal) -> bool:
|
|
||||||
return (
|
|
||||||
observation.availability == Availability.IN_STOCK
|
|
||||||
and observation.total_price is not None
|
|
||||||
and observation.confidence >= minimum_confidence
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def daily_series(
|
|
||||||
observations: Sequence[PriceObservation],
|
|
||||||
*,
|
|
||||||
window_start: date,
|
|
||||||
window_end_exclusive: date,
|
|
||||||
minimum_confidence: Decimal,
|
|
||||||
) -> tuple[tuple[Decimal, ...], tuple[UUID, ...]]:
|
|
||||||
by_day: dict[date, PriceObservation] = {}
|
|
||||||
for item in observations:
|
|
||||||
day = item.observed_at.date()
|
|
||||||
if day < window_start or day >= window_end_exclusive:
|
|
||||||
continue
|
|
||||||
current = by_day.get(day)
|
|
||||||
if current is None or (item.observed_at, item.ingested_at, item.id) > (
|
|
||||||
current.observed_at,
|
|
||||||
current.ingested_at,
|
|
||||||
current.id,
|
|
||||||
):
|
|
||||||
by_day[day] = item
|
|
||||||
values: list[Decimal] = []
|
|
||||||
ids: list[UUID] = []
|
|
||||||
for day in sorted(by_day):
|
|
||||||
observation = by_day[day]
|
|
||||||
if _quality_ok(observation, minimum_confidence):
|
|
||||||
values.append(observation.total_price or Decimal("0"))
|
|
||||||
ids.append(observation.id)
|
|
||||||
return tuple(values), tuple(ids)
|
|
||||||
|
|
||||||
|
|
||||||
def build_offer_reference_history(
|
|
||||||
observations: Sequence[PriceObservation],
|
|
||||||
current: PriceObservation,
|
|
||||||
model: ScoringModel,
|
|
||||||
) -> dict[int, HistoryWindow]:
|
|
||||||
current_day = current.observed_at.date()
|
|
||||||
windows: dict[int, HistoryWindow] = {}
|
|
||||||
for days in (30, 90, 365):
|
|
||||||
values, ids = daily_series(
|
|
||||||
observations,
|
|
||||||
window_start=current_day - timedelta(days=days),
|
|
||||||
window_end_exclusive=current_day,
|
|
||||||
minimum_confidence=model.minimum_confidence,
|
|
||||||
)
|
|
||||||
windows[days] = HistoryWindow(days=days, values=values, observation_ids=ids)
|
|
||||||
return windows
|
|
||||||
|
|
||||||
|
|
||||||
def score_price(
|
|
||||||
*,
|
|
||||||
current_price: Decimal,
|
|
||||||
reference_history: Sequence[Decimal],
|
|
||||||
budget: Budget,
|
|
||||||
model: ScoringModel,
|
|
||||||
) -> tuple[Decimal, Decimal, Decimal | None, PriceBasis, str | None]:
|
|
||||||
"""Pure price score over current_price, reference_history and budget."""
|
|
||||||
if current_price <= budget.target:
|
|
||||||
budget_score = Decimal("100")
|
|
||||||
else:
|
|
||||||
budget_score = clamp(
|
|
||||||
Decimal("100") * (budget.maximum - current_price) / (budget.maximum - budget.target)
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(reference_history) < model.minimum_history_days:
|
|
||||||
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
|
|
||||||
|
|
||||||
reference = median(reference_history)
|
|
||||||
if reference is None or reference == 0:
|
|
||||||
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
|
|
||||||
|
|
||||||
discount = (reference - current_price) / reference
|
|
||||||
history_score = clamp(
|
|
||||||
Decimal("50") + Decimal("50") * discount / model.history_discount_for_max_score
|
|
||||||
)
|
|
||||||
price_score = model.budget_weight * budget_score + model.history_weight * history_score
|
|
||||||
return price_score, budget_score, history_score, PriceBasis.BUDGET_AND_HISTORY, None
|
|
||||||
|
|
||||||
|
|
||||||
def price_statistics_from_windows(
|
|
||||||
current: PriceObservation | None,
|
|
||||||
windows: dict[int, HistoryWindow],
|
|
||||||
) -> PriceStatistics:
|
|
||||||
window_30 = windows.get(30, HistoryWindow(30, (), ()))
|
|
||||||
window_90 = windows.get(90, HistoryWindow(90, (), ()))
|
|
||||||
window_365 = windows.get(365, HistoryWindow(365, (), ()))
|
|
||||||
used = window_30.observation_ids + window_90.observation_ids + window_365.observation_ids
|
|
||||||
if current is not None:
|
|
||||||
used = (current.id,) + used
|
|
||||||
lowest = min(window_365.values) if window_365.values else None
|
|
||||||
return PriceStatistics(
|
|
||||||
current_total=None if current is None else current.total_price,
|
|
||||||
median_30d=median(window_30.values),
|
|
||||||
median_30d_days=window_30.day_count,
|
|
||||||
median_90d=median(window_90.values),
|
|
||||||
median_90d_days=window_90.day_count,
|
|
||||||
lowest_365d=lowest,
|
|
||||||
lowest_365d_days=window_365.day_count,
|
|
||||||
lowest_365d_is_full_year=window_365.day_count >= 365,
|
|
||||||
used_observation_ids=tuple(dict.fromkeys(used)),
|
|
||||||
current_observation_id=None if current is None else current.id,
|
|
||||||
)
|
|
||||||
|
|
@ -1,86 +0,0 @@
|
||||||
from decimal import Decimal
|
|
||||||
|
|
||||||
from app.domain.attributes import statement_is_usable
|
|
||||||
from app.domain.enums import ConstraintStatus, PreferenceDirection
|
|
||||||
from app.domain.models import (
|
|
||||||
AttributeStatement,
|
|
||||||
ConstraintResult,
|
|
||||||
Preference,
|
|
||||||
ScoreComponent,
|
|
||||||
ScoringModel,
|
|
||||||
)
|
|
||||||
from app.scoring.mathutil import as_decimal, clamp
|
|
||||||
|
|
||||||
|
|
||||||
def score_numeric_linear(value: Decimal, acceptable: Decimal, ideal: Decimal, direction: PreferenceDirection) -> Decimal:
|
|
||||||
if direction == PreferenceDirection.HIGHER:
|
|
||||||
return clamp(Decimal("50") + Decimal("50") * (value - acceptable) / (ideal - acceptable))
|
|
||||||
return clamp(Decimal("50") + Decimal("50") * (acceptable - value) / (acceptable - ideal))
|
|
||||||
|
|
||||||
|
|
||||||
def score_product(
|
|
||||||
preferences: tuple[Preference, ...],
|
|
||||||
snapshot: dict[str, AttributeStatement],
|
|
||||||
constraint_results: tuple[ConstraintResult, ...],
|
|
||||||
model: ScoringModel,
|
|
||||||
) -> tuple[Decimal, Decimal, tuple[ScoreComponent, ...]]:
|
|
||||||
components: list[ScoreComponent] = []
|
|
||||||
weighted = Decimal("0")
|
|
||||||
total_weight = sum((pref.weight for pref in preferences), Decimal("0"))
|
|
||||||
usable_weight = Decimal("0")
|
|
||||||
|
|
||||||
for preference in preferences:
|
|
||||||
statement = snapshot.get(preference.attribute)
|
|
||||||
usable = statement_is_usable(statement, model.minimum_confidence)
|
|
||||||
if usable:
|
|
||||||
raw = score_numeric_linear(
|
|
||||||
as_decimal(statement.normalized_value),
|
|
||||||
preference.acceptable,
|
|
||||||
preference.ideal,
|
|
||||||
preference.direction,
|
|
||||||
)
|
|
||||||
usable_weight += preference.weight
|
|
||||||
else:
|
|
||||||
raw = model.missing_preference_score
|
|
||||||
contribution = preference.weight * raw
|
|
||||||
weighted += contribution
|
|
||||||
components.append(
|
|
||||||
ScoreComponent(
|
|
||||||
kind="preference",
|
|
||||||
component_key=preference.id,
|
|
||||||
raw_score=raw,
|
|
||||||
weight=preference.weight,
|
|
||||||
contribution=contribution,
|
|
||||||
details={
|
|
||||||
"attribute": preference.attribute,
|
|
||||||
"strategy": preference.strategy.value,
|
|
||||||
"direction": preference.direction.value,
|
|
||||||
"acceptable": preference.acceptable,
|
|
||||||
"ideal": preference.ideal,
|
|
||||||
"usable": usable,
|
|
||||||
"attribute_row_id": None if statement is None else str(statement.id),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
base = weighted / total_weight if total_weight else Decimal("0")
|
|
||||||
optional_issues = [
|
|
||||||
result
|
|
||||||
for result in constraint_results
|
|
||||||
if not result.mandatory and result.status in {ConstraintStatus.FAIL.value, ConstraintStatus.UNKNOWN.value}
|
|
||||||
]
|
|
||||||
penalty = model.optional_constraint_penalty_points * Decimal(len(optional_issues))
|
|
||||||
product_score = clamp(base - penalty)
|
|
||||||
if optional_issues:
|
|
||||||
components.append(
|
|
||||||
ScoreComponent(
|
|
||||||
kind="optional_penalty",
|
|
||||||
component_key="optional_constraints",
|
|
||||||
raw_score=penalty,
|
|
||||||
weight=None,
|
|
||||||
contribution=-penalty,
|
|
||||||
details={"count": len(optional_issues), "constraint_ids": [item.constraint_id for item in optional_issues]},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
coverage = (usable_weight / total_weight) if total_weight else Decimal("0")
|
|
||||||
return product_score, coverage, tuple(components)
|
|
||||||
|
|
@ -1,334 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from uuid import uuid4
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.discovery.enums import FilterType, MappingStatus, TaxonomyStatus
|
|
||||||
from app.domain.serialize import jsonable
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.persistence.discovery_models import (
|
|
||||||
CriterionDefinitionRecord,
|
|
||||||
CriterionMappingRecord,
|
|
||||||
ProductCategoryRecord,
|
|
||||||
SourceCategoryRecord,
|
|
||||||
SourceFilterDefinitionRecord,
|
|
||||||
SourceFilterOptionRecord,
|
|
||||||
SourceFilterRangeRecord,
|
|
||||||
)
|
|
||||||
from app.discovery.definitions import upsert_definition
|
|
||||||
from app.persistence.repositories import CatalogRepository
|
|
||||||
|
|
||||||
|
|
||||||
FILTER_OPERATORS = {
|
|
||||||
FilterType.BOOLEAN.value: ["=", "!="],
|
|
||||||
FilterType.ENUM.value: ["=", "!="],
|
|
||||||
FilterType.MULTI_ENUM.value: ["in", "not_in"],
|
|
||||||
FilterType.NUMERIC_RANGE.value: [">=", "<=", "between"],
|
|
||||||
FilterType.NUMERIC_THRESHOLD.value: [">=", ">", "<=", "<"],
|
|
||||||
FilterType.TEXT.value: ["=", "contains"],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class CatalogService:
|
|
||||||
def __init__(self, session: Session) -> None:
|
|
||||||
self.session = session
|
|
||||||
self.catalog = CatalogRepository(session)
|
|
||||||
|
|
||||||
def list_product_categories(self) -> list[dict[str, Any]]:
|
|
||||||
rows = self.session.scalars(select(ProductCategoryRecord).order_by(ProductCategoryRecord.name)).all()
|
|
||||||
return [{"id": row.id, "name": row.name, "status": row.status} for row in rows]
|
|
||||||
|
|
||||||
def get_product_category(self, category_id: str) -> ProductCategoryRecord:
|
|
||||||
row = self.session.get(ProductCategoryRecord, category_id)
|
|
||||||
if row is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "product category not found", {"id": category_id})
|
|
||||||
return row
|
|
||||||
|
|
||||||
def sources_for_category(self, product_category_id: str) -> list[dict[str, Any]]:
|
|
||||||
self.get_product_category(product_category_id)
|
|
||||||
rows = self.session.scalars(
|
|
||||||
select(SourceCategoryRecord)
|
|
||||||
.where(SourceCategoryRecord.product_category_id == product_category_id)
|
|
||||||
.order_by(SourceCategoryRecord.source_id)
|
|
||||||
).all()
|
|
||||||
sources = {item.id: item for item in self.catalog.list_sources()}
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"source": row.source_id,
|
|
||||||
"source_category_id": row.id,
|
|
||||||
"display_name": sources[row.source_id].name if row.source_id in sources else row.name,
|
|
||||||
"external_category_key": row.external_category_key,
|
|
||||||
"status": row.status,
|
|
||||||
"taxonomy_status": row.taxonomy_status,
|
|
||||||
}
|
|
||||||
for row in rows
|
|
||||||
]
|
|
||||||
|
|
||||||
def register_source_category(self, source_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
if self.catalog.get_source(source_id) is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
|
||||||
from app.sources.geizhals.category_url import parse_category_key, slugify
|
|
||||||
from app.services.discovery_service import TaxonomyService
|
|
||||||
|
|
||||||
source_url = str(payload["source_url"]).strip()
|
|
||||||
category_key = parse_category_key(source_url)
|
|
||||||
name = str(payload.get("name") or category_key)
|
|
||||||
product_category_id = str(payload.get("product_category_id") or slugify(name))
|
|
||||||
ensure_product_category(self.session, product_category_id, str(payload.get("product_category_name") or name))
|
|
||||||
category = ensure_source_category(
|
|
||||||
self.session,
|
|
||||||
source_id=source_id,
|
|
||||||
external_category_key=category_key,
|
|
||||||
product_category_id=product_category_id,
|
|
||||||
name=name,
|
|
||||||
source_url=source_url,
|
|
||||||
)
|
|
||||||
category.source_url = source_url
|
|
||||||
category.name = name
|
|
||||||
category.product_category_id = product_category_id
|
|
||||||
self.session.flush()
|
|
||||||
TaxonomyService(self.session).discover(source_id, category_key)
|
|
||||||
items = self.source_categories(source_id)
|
|
||||||
return next(item for item in items if item["external_category_key"] == category_key)
|
|
||||||
|
|
||||||
def source_categories(self, source_id: str) -> list[dict[str, Any]]:
|
|
||||||
if self.catalog.get_source(source_id) is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
|
||||||
rows = self.session.scalars(
|
|
||||||
select(SourceCategoryRecord)
|
|
||||||
.where(SourceCategoryRecord.source_id == source_id)
|
|
||||||
.order_by(SourceCategoryRecord.external_category_key)
|
|
||||||
).all()
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"id": row.id,
|
|
||||||
"source": row.source_id,
|
|
||||||
"source_category_id": row.id,
|
|
||||||
"external_category_key": row.external_category_key,
|
|
||||||
"name": row.name,
|
|
||||||
"product_category_id": row.product_category_id,
|
|
||||||
"status": row.status,
|
|
||||||
"taxonomy_status": row.taxonomy_status,
|
|
||||||
"source_url": row.source_url,
|
|
||||||
}
|
|
||||||
for row in rows
|
|
||||||
]
|
|
||||||
|
|
||||||
def available_criteria(self, product_category_id: str, source_id: str) -> dict[str, Any]:
|
|
||||||
self.get_product_category(product_category_id)
|
|
||||||
if self.catalog.get_source(source_id) is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
|
||||||
source_category = self.session.scalars(
|
|
||||||
select(SourceCategoryRecord).where(
|
|
||||||
SourceCategoryRecord.source_id == source_id,
|
|
||||||
SourceCategoryRecord.product_category_id == product_category_id,
|
|
||||||
)
|
|
||||||
).first()
|
|
||||||
if source_category is not None and self._should_learn(source_category, source_id):
|
|
||||||
from app.services.discovery_service import TaxonomyService
|
|
||||||
|
|
||||||
TaxonomyService(self.session).discover(source_id, source_category.external_category_key)
|
|
||||||
source_category = self.session.scalars(
|
|
||||||
select(SourceCategoryRecord).where(
|
|
||||||
SourceCategoryRecord.source_id == source_id,
|
|
||||||
SourceCategoryRecord.product_category_id == product_category_id,
|
|
||||||
)
|
|
||||||
).first()
|
|
||||||
criteria: list[dict[str, Any]] = []
|
|
||||||
if source_category is not None:
|
|
||||||
filters = self.session.scalars(
|
|
||||||
select(SourceFilterDefinitionRecord)
|
|
||||||
.where(SourceFilterDefinitionRecord.source_category_id == source_category.id)
|
|
||||||
.order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label)
|
|
||||||
).all()
|
|
||||||
mappings = {
|
|
||||||
row.external_filter_key: row
|
|
||||||
for row in self.session.scalars(
|
|
||||||
select(CriterionMappingRecord).where(
|
|
||||||
CriterionMappingRecord.source_category_id == source_category.id
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
for filt in filters:
|
|
||||||
mapping = mappings.get(filt.external_filter_key)
|
|
||||||
definition = None if mapping is None else self.session.get(CriterionDefinitionRecord, mapping.criterion_definition_id)
|
|
||||||
if definition is None:
|
|
||||||
definition = upsert_definition(
|
|
||||||
self.session,
|
|
||||||
{
|
|
||||||
"key": f"{source_id}:{source_category.external_category_key}:{filt.external_filter_key}",
|
|
||||||
"origin": "SOURCE",
|
|
||||||
"source": source_id,
|
|
||||||
"product_category_id": product_category_id,
|
|
||||||
"label": filt.label,
|
|
||||||
"value_type": "number"
|
|
||||||
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
|
|
||||||
else "string",
|
|
||||||
"unit": filt.canonical_unit,
|
|
||||||
"category": product_category_id,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
self.session.flush()
|
|
||||||
options = self.session.scalars(
|
|
||||||
select(SourceFilterOptionRecord)
|
|
||||||
.where(SourceFilterOptionRecord.filter_id == filt.id)
|
|
||||||
.order_by(SourceFilterOptionRecord.source_value)
|
|
||||||
).all()
|
|
||||||
rng = self.session.get(SourceFilterRangeRecord, filt.id)
|
|
||||||
criteria.append(
|
|
||||||
self._criterion_out(
|
|
||||||
definition,
|
|
||||||
filt,
|
|
||||||
mapping,
|
|
||||||
options,
|
|
||||||
rng,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
internals = self.session.scalars(
|
|
||||||
select(CriterionDefinitionRecord)
|
|
||||||
.where(
|
|
||||||
CriterionDefinitionRecord.origin == "INTERNAL",
|
|
||||||
(CriterionDefinitionRecord.product_category_id == product_category_id)
|
|
||||||
| (CriterionDefinitionRecord.category == product_category_id),
|
|
||||||
)
|
|
||||||
.order_by(CriterionDefinitionRecord.label)
|
|
||||||
).all()
|
|
||||||
seen = {item["criterion_definition_id"] for item in criteria}
|
|
||||||
for definition in internals:
|
|
||||||
if definition.id in seen:
|
|
||||||
continue
|
|
||||||
criteria.append(self._criterion_out(definition, None, None, [], None))
|
|
||||||
return jsonable(
|
|
||||||
{
|
|
||||||
"product_category_id": product_category_id,
|
|
||||||
"source": source_id,
|
|
||||||
"taxonomy_status": None if source_category is None else source_category.taxonomy_status,
|
|
||||||
"criteria": criteria,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool:
|
|
||||||
if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
|
|
||||||
return True
|
|
||||||
if source_category.taxonomy_status != TaxonomyStatus.READY.value:
|
|
||||||
return False
|
|
||||||
from app.settings import get_settings
|
|
||||||
|
|
||||||
if get_settings().geizhals_use_fixtures or source_id != "geizhals":
|
|
||||||
return False
|
|
||||||
keys = {
|
|
||||||
row.external_filter_key
|
|
||||||
for row in self.session.scalars(
|
|
||||||
select(SourceFilterDefinitionRecord).where(
|
|
||||||
SourceFilterDefinitionRecord.source_category_id == source_category.id
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return len(keys) < 40
|
|
||||||
|
|
||||||
def list_templates(self) -> list[dict[str, Any]]:
|
|
||||||
rows, _total = self.catalog.list_profiles(200, 0)
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"id": row.id,
|
|
||||||
"version": row.version,
|
|
||||||
"name": row.definition.get("name") or row.id,
|
|
||||||
"product_category_id": row.definition.get("category"),
|
|
||||||
"market": row.definition.get("market"),
|
|
||||||
"currency": row.definition.get("currency"),
|
|
||||||
"description": row.definition.get("description"),
|
|
||||||
}
|
|
||||||
for row in rows
|
|
||||||
]
|
|
||||||
|
|
||||||
def _criterion_out(
|
|
||||||
self,
|
|
||||||
definition: CriterionDefinitionRecord,
|
|
||||||
filt: SourceFilterDefinitionRecord | None,
|
|
||||||
mapping: CriterionMappingRecord | None,
|
|
||||||
options: list[SourceFilterOptionRecord],
|
|
||||||
rng: SourceFilterRangeRecord | None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
filter_type = None if filt is None else filt.filter_type
|
|
||||||
return {
|
|
||||||
"criterion_definition_id": definition.id,
|
|
||||||
"label": definition.label,
|
|
||||||
"group_label": None if filt is None else (filt.group_label or "Allgemein"),
|
|
||||||
"description": definition.description,
|
|
||||||
"origin": definition.origin,
|
|
||||||
"source": definition.source,
|
|
||||||
"product_category_id": definition.product_category_id,
|
|
||||||
"filter_type": filter_type,
|
|
||||||
"unit": definition.unit or (None if filt is None else filt.canonical_unit),
|
|
||||||
"operators": FILTER_OPERATORS.get(filter_type or "", ["=", ">=", "<="]),
|
|
||||||
"options": [
|
|
||||||
{
|
|
||||||
"source_value": option.source_value,
|
|
||||||
"normalized_value": option.normalized_value,
|
|
||||||
"operator": option.operator,
|
|
||||||
"unit": option.unit,
|
|
||||||
}
|
|
||||||
for option in options
|
|
||||||
],
|
|
||||||
"range": None
|
|
||||||
if rng is None
|
|
||||||
else {"minimum": rng.minimum, "maximum": rng.maximum, "step": rng.step, "unit": rng.unit},
|
|
||||||
"mapping_status": None if mapping is None else mapping.status,
|
|
||||||
"supports_discovery": mapping is not None
|
|
||||||
and mapping.status == MappingStatus.VERIFIED.value
|
|
||||||
and filt is not None,
|
|
||||||
"scoring_attribute_key": definition.scoring_attribute_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_product_category(session: Session, category_id: str, name: str | None = None) -> ProductCategoryRecord:
|
|
||||||
row = session.get(ProductCategoryRecord, category_id)
|
|
||||||
if row is not None:
|
|
||||||
return row
|
|
||||||
row = ProductCategoryRecord(id=category_id, name=name or category_id.title(), status="ACTIVE")
|
|
||||||
session.add(row)
|
|
||||||
session.flush()
|
|
||||||
return row
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_source_category(
|
|
||||||
session: Session,
|
|
||||||
*,
|
|
||||||
source_id: str,
|
|
||||||
external_category_key: str,
|
|
||||||
product_category_id: str,
|
|
||||||
name: str,
|
|
||||||
source_url: str,
|
|
||||||
) -> SourceCategoryRecord:
|
|
||||||
row = session.scalars(
|
|
||||||
select(SourceCategoryRecord).where(
|
|
||||||
SourceCategoryRecord.source_id == source_id,
|
|
||||||
SourceCategoryRecord.external_category_key == external_category_key,
|
|
||||||
)
|
|
||||||
).first()
|
|
||||||
if row is None:
|
|
||||||
row = SourceCategoryRecord(
|
|
||||||
id=uuid4(),
|
|
||||||
source_id=source_id,
|
|
||||||
external_category_key=external_category_key,
|
|
||||||
name=name,
|
|
||||||
source_url=source_url,
|
|
||||||
product_category_id=product_category_id,
|
|
||||||
canonical_category=product_category_id,
|
|
||||||
status="ACTIVE",
|
|
||||||
taxonomy_status=TaxonomyStatus.NOT_LEARNED.value,
|
|
||||||
last_taxonomy_scan_at=None,
|
|
||||||
last_verified_at=None,
|
|
||||||
)
|
|
||||||
session.add(row)
|
|
||||||
session.flush()
|
|
||||||
return row
|
|
||||||
if row.product_category_id is None:
|
|
||||||
row.product_category_id = product_category_id
|
|
||||||
if not row.source_url:
|
|
||||||
row.source_url = source_url
|
|
||||||
session.flush()
|
|
||||||
return row
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,210 +0,0 @@
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID, uuid4
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.clock import now_utc
|
|
||||||
from app.domain.serialize import jsonable
|
|
||||||
from app.domain.timeutil import require_utc, to_iso_z
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.persistence import models as orm
|
|
||||||
from app.persistence.db import engine
|
|
||||||
from app.persistence.repositories import CatalogRepository
|
|
||||||
from app.scoring.engine import evaluate_deal
|
|
||||||
from app.scoring.mathutil import round_score
|
|
||||||
from app.settings import get_settings
|
|
||||||
from sqlalchemy.orm import Session as SASession
|
|
||||||
from sqlalchemy.orm import sessionmaker
|
|
||||||
|
|
||||||
|
|
||||||
def evaluate_and_store(
|
|
||||||
*,
|
|
||||||
variant_id: UUID,
|
|
||||||
payload: dict[str, Any],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
now = now_utc()
|
|
||||||
as_of = payload.get("as_of")
|
|
||||||
if as_of is None:
|
|
||||||
as_of = now
|
|
||||||
else:
|
|
||||||
as_of = require_utc(as_of, "as_of")
|
|
||||||
if as_of > now:
|
|
||||||
raise AppError(422, "AS_OF_IN_FUTURE", "as_of must not be in the future")
|
|
||||||
snapshot_cutoff = now
|
|
||||||
|
|
||||||
maker = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
|
||||||
with engine.connect().execution_options(isolation_level="REPEATABLE READ") as connection:
|
|
||||||
with SASession(bind=connection) as session:
|
|
||||||
with session.begin():
|
|
||||||
repo = CatalogRepository(session)
|
|
||||||
variant = repo.get_variant(variant_id)
|
|
||||||
if variant is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
|
||||||
offer = repo.get_offer(payload["offer_id"])
|
|
||||||
if offer is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(payload["offer_id"])})
|
|
||||||
if offer.variant_id != variant_id:
|
|
||||||
raise AppError(422, "OFFER_VARIANT_MISMATCH", "offer does not belong to the variant")
|
|
||||||
product = repo.get_product(variant.product_id)
|
|
||||||
if product is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "product not found")
|
|
||||||
profile = repo.get_profile(payload["profile_id"], int(payload["profile_version"]))
|
|
||||||
if profile is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "profile version not found")
|
|
||||||
model = repo.get_scoring_model(profile.scoring_version)
|
|
||||||
if model is None:
|
|
||||||
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version is missing")
|
|
||||||
attributes = tuple(repo.list_attributes(variant_id))
|
|
||||||
observations, _ = repo.list_observations(offer.id, limit=None)
|
|
||||||
result = evaluate_deal(
|
|
||||||
profile=profile,
|
|
||||||
scoring_model=model,
|
|
||||||
product=product,
|
|
||||||
variant=variant,
|
|
||||||
offer=offer,
|
|
||||||
attributes=attributes,
|
|
||||||
observations=tuple(observations),
|
|
||||||
definitions=repo.all_definitions(),
|
|
||||||
as_of=as_of,
|
|
||||||
snapshot_cutoff=snapshot_cutoff,
|
|
||||||
engine_build=get_settings().engine_build,
|
|
||||||
)
|
|
||||||
response = evaluation_to_response(result, evaluation_id=uuid4(), created_at=now)
|
|
||||||
record = orm.EvaluationRecord(
|
|
||||||
id=response["id"],
|
|
||||||
variant_id=variant.id,
|
|
||||||
offer_id=offer.id,
|
|
||||||
profile_id=profile.id,
|
|
||||||
profile_version=profile.version,
|
|
||||||
scoring_version=model.version,
|
|
||||||
as_of=as_of,
|
|
||||||
snapshot_cutoff=snapshot_cutoff,
|
|
||||||
scoring_hash=model.definition_hash,
|
|
||||||
engine_build=result.engine_build,
|
|
||||||
product_eligibility=result.product_eligibility.value,
|
|
||||||
offer_eligibility=result.offer_eligibility.value,
|
|
||||||
overall_eligibility=result.overall_eligibility.value,
|
|
||||||
product_score=result.product_score,
|
|
||||||
price_score=result.price_score,
|
|
||||||
deal_score=result.deal_score,
|
|
||||||
classification=result.classification,
|
|
||||||
price_reference_scope=result.price_reference_scope.value,
|
|
||||||
input_snapshot=result.input_snapshot,
|
|
||||||
result_snapshot=response,
|
|
||||||
created_at=now,
|
|
||||||
)
|
|
||||||
repo.add_evaluation(record)
|
|
||||||
for component in result.components:
|
|
||||||
repo.add_component(
|
|
||||||
orm.EvaluationComponentRecord(
|
|
||||||
id=uuid4(),
|
|
||||||
evaluation_id=record.id,
|
|
||||||
kind=component.kind,
|
|
||||||
component_key=component.component_key,
|
|
||||||
raw_score=component.raw_score,
|
|
||||||
weight=component.weight,
|
|
||||||
contribution=component.contribution,
|
|
||||||
details=jsonable(component.details),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return response
|
|
||||||
|
|
||||||
|
|
||||||
def load_evaluation(session: Session, evaluation_id: UUID) -> dict[str, Any]:
|
|
||||||
record = CatalogRepository(session).get_evaluation(evaluation_id)
|
|
||||||
if record is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "evaluation not found", {"id": str(evaluation_id)})
|
|
||||||
return record.result_snapshot
|
|
||||||
|
|
||||||
|
|
||||||
def evaluation_to_response(result, *, evaluation_id: UUID, created_at: datetime) -> dict[str, Any]:
|
|
||||||
stats = result.price_statistics
|
|
||||||
return {
|
|
||||||
"id": str(evaluation_id),
|
|
||||||
"profile_id": result.profile_id,
|
|
||||||
"profile_version": result.profile_version,
|
|
||||||
"scoring_version": result.scoring_version,
|
|
||||||
"variant_id": str(result.variant_id),
|
|
||||||
"offer_id": str(result.offer_id),
|
|
||||||
"as_of": to_iso_z(result.as_of),
|
|
||||||
"snapshot_cutoff": to_iso_z(result.snapshot_cutoff),
|
|
||||||
"scoring_hash": result.scoring_hash,
|
|
||||||
"engine_build": result.engine_build,
|
|
||||||
"product_eligibility": result.product_eligibility.value,
|
|
||||||
"offer_eligibility": result.offer_eligibility.value,
|
|
||||||
"overall_eligibility": result.overall_eligibility.value,
|
|
||||||
"eligible": result.eligible,
|
|
||||||
"product_score": _num(result.product_score),
|
|
||||||
"price_score": _num(result.price_score),
|
|
||||||
"deal_score": _num(result.deal_score),
|
|
||||||
"classification": result.classification,
|
|
||||||
"price_basis": None if result.price_basis is None else result.price_basis.value,
|
|
||||||
"product_coverage": _num(result.product_coverage),
|
|
||||||
"failed_constraints": [_constraint(item) for item in result.failed_constraints],
|
|
||||||
"unknown_constraints": [_constraint(item) for item in result.unknown_constraints],
|
|
||||||
"constraint_results": [_constraint(item) for item in result.constraint_results],
|
|
||||||
"product_gates": [_gate(item) for item in result.product_gates],
|
|
||||||
"offer_gates": [_gate(item) for item in result.offer_gates],
|
|
||||||
"components": [_component(item) for item in result.components],
|
|
||||||
"price_statistics": {
|
|
||||||
"current_total": _money(stats.current_total),
|
|
||||||
"median_30d": _money(stats.median_30d),
|
|
||||||
"median_30d_days": stats.median_30d_days,
|
|
||||||
"median_90d": _money(stats.median_90d),
|
|
||||||
"median_90d_days": stats.median_90d_days,
|
|
||||||
"lowest_365d": _money(stats.lowest_365d),
|
|
||||||
"lowest_365d_days": stats.lowest_365d_days,
|
|
||||||
"lowest_365d_is_full_year": stats.lowest_365d_is_full_year,
|
|
||||||
"used_observation_ids": [str(item) for item in stats.used_observation_ids],
|
|
||||||
"current_observation_id": None if stats.current_observation_id is None else str(stats.current_observation_id),
|
|
||||||
},
|
|
||||||
"price_reference_scope": result.price_reference_scope.value,
|
|
||||||
"reasons": [{"code": item.code, "message": item.message} for item in result.reasons],
|
|
||||||
"warnings": list(result.warnings),
|
|
||||||
"input_snapshot": result.input_snapshot,
|
|
||||||
"created_at": to_iso_z(created_at),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _num(value: Decimal | None) -> float | None:
|
|
||||||
return None if value is None else float(round_score(value))
|
|
||||||
|
|
||||||
|
|
||||||
def _money(value: Decimal | None) -> str | None:
|
|
||||||
return None if value is None else format(value.quantize(Decimal("0.01")), "f")
|
|
||||||
|
|
||||||
|
|
||||||
def _constraint(item) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"constraint_id": item.constraint_id,
|
|
||||||
"attribute": item.attribute,
|
|
||||||
"operator": item.operator,
|
|
||||||
"expected": jsonable(item.expected),
|
|
||||||
"actual": jsonable(item.actual),
|
|
||||||
"status": item.status,
|
|
||||||
"mandatory": item.mandatory,
|
|
||||||
"attribute_row_id": None if item.attribute_row_id is None else str(item.attribute_row_id),
|
|
||||||
"reason_code": item.reason_code,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _gate(item) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"key": item.key,
|
|
||||||
"status": item.status,
|
|
||||||
"reason_code": item.reason_code,
|
|
||||||
"details": jsonable(item.details),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _component(item) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"kind": item.kind,
|
|
||||||
"component_key": item.component_key,
|
|
||||||
"raw_score": None if item.raw_score is None else float(round_score(item.raw_score)),
|
|
||||||
"weight": None if item.weight is None else float(item.weight),
|
|
||||||
"contribution": None if item.contribution is None else float(round_score(item.contribution)),
|
|
||||||
"details": jsonable(item.details),
|
|
||||||
}
|
|
||||||
|
|
@ -1,94 +0,0 @@
|
||||||
from app.errors import AppError
|
|
||||||
from app.services.ingestion_service import IngestionService
|
|
||||||
from app.sources.base import ProductSource
|
|
||||||
|
|
||||||
|
|
||||||
class ImportService:
|
|
||||||
def __init__(self, ingestion: IngestionService) -> None:
|
|
||||||
self.ingestion = ingestion
|
|
||||||
|
|
||||||
def import_product(self, source: ProductSource, external_id: str) -> None:
|
|
||||||
product = source.get_product(external_id)
|
|
||||||
try:
|
|
||||||
self.ingestion.create_product(
|
|
||||||
{
|
|
||||||
"id": product.id,
|
|
||||||
"category": product.category,
|
|
||||||
"brand": product.brand,
|
|
||||||
"name": product.name,
|
|
||||||
"model": product.model,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
except AppError as exc:
|
|
||||||
if exc.status_code != 409:
|
|
||||||
raise
|
|
||||||
for variant in product.variants:
|
|
||||||
try:
|
|
||||||
self.ingestion.create_variant(
|
|
||||||
product.id,
|
|
||||||
{
|
|
||||||
"id": variant.id,
|
|
||||||
"name": variant.name,
|
|
||||||
"manufacturer_sku": variant.manufacturer_sku,
|
|
||||||
"identifiers": variant.identifiers,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
except AppError as exc:
|
|
||||||
if exc.status_code != 409:
|
|
||||||
raise
|
|
||||||
for attribute in variant.attributes:
|
|
||||||
try:
|
|
||||||
self.ingestion.add_attribute(
|
|
||||||
variant.id,
|
|
||||||
{
|
|
||||||
"id": attribute.id,
|
|
||||||
"attribute": attribute.attribute,
|
|
||||||
"raw_value": attribute.raw_value,
|
|
||||||
"normalized_value": attribute.normalized_value,
|
|
||||||
"unit": attribute.unit,
|
|
||||||
"normalization_status": attribute.normalization_status,
|
|
||||||
"normalizer_version": attribute.normalizer_version,
|
|
||||||
"confidence": attribute.confidence,
|
|
||||||
"source_id": source.source_key,
|
|
||||||
"source_reference": attribute.source_reference,
|
|
||||||
"observed_at": attribute.observed_at,
|
|
||||||
"normalization_error": attribute.normalization_error,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
except AppError as exc:
|
|
||||||
if exc.status_code != 409:
|
|
||||||
raise
|
|
||||||
for offer in source.get_offers(external_id):
|
|
||||||
try:
|
|
||||||
self.ingestion.create_offer(
|
|
||||||
offer.variant_id,
|
|
||||||
{
|
|
||||||
"id": offer.id,
|
|
||||||
"source_id": source.source_key,
|
|
||||||
"external_id": offer.external_id,
|
|
||||||
"seller": offer.seller,
|
|
||||||
"url": offer.url,
|
|
||||||
"market": offer.market,
|
|
||||||
"currency": offer.currency,
|
|
||||||
"condition": offer.condition,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
except AppError as exc:
|
|
||||||
if exc.status_code != 409:
|
|
||||||
raise
|
|
||||||
for price in source.get_price_history(offer.external_id):
|
|
||||||
self.ingestion.add_price(
|
|
||||||
offer.id,
|
|
||||||
{
|
|
||||||
"id": price.id,
|
|
||||||
"source_id": source.source_key,
|
|
||||||
"external_observation_id": price.external_observation_id,
|
|
||||||
"observed_at": price.observed_at,
|
|
||||||
"item_price": price.item_price,
|
|
||||||
"shipping_price": price.shipping_price,
|
|
||||||
"currency": price.currency,
|
|
||||||
"availability": price.availability,
|
|
||||||
"raw_payload": price.raw_payload,
|
|
||||||
"confidence": price.confidence,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
@ -1,263 +0,0 @@
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID, uuid4
|
|
||||||
|
|
||||||
from sqlalchemy.exc import IntegrityError
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
|
|
||||||
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
|
|
||||||
from app.clock import now_utc
|
|
||||||
from app.domain.timeutil import require_utc
|
|
||||||
from app.domain.validation import DomainValidationError, parse_profile, validate_attribute_key, validate_currency
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.normalization.manual import ManualNormalizer
|
|
||||||
from app.persistence import models as orm
|
|
||||||
from app.persistence.mapping import to_observation
|
|
||||||
from app.persistence.repositories import CatalogRepository
|
|
||||||
|
|
||||||
|
|
||||||
def _now() -> datetime:
|
|
||||||
return now_utc()
|
|
||||||
|
|
||||||
|
|
||||||
def _uuid(value: UUID | None) -> UUID:
|
|
||||||
return value or uuid4()
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionService:
|
|
||||||
def __init__(self, session: Session) -> None:
|
|
||||||
self.session = session
|
|
||||||
self.repo = CatalogRepository(session)
|
|
||||||
self.normalizer = ManualNormalizer()
|
|
||||||
|
|
||||||
def create_definition(self, payload: dict[str, Any]) -> AttributeDefinition:
|
|
||||||
key = str(payload["key"])
|
|
||||||
validate_attribute_key(key)
|
|
||||||
row = orm.AttributeDefinitionRecord(
|
|
||||||
key=key,
|
|
||||||
value_type=ValueType(payload["value_type"]).value,
|
|
||||||
canonical_unit=payload.get("canonical_unit"),
|
|
||||||
minimum=payload.get("minimum"),
|
|
||||||
maximum=payload.get("maximum"),
|
|
||||||
description=payload.get("description"),
|
|
||||||
)
|
|
||||||
if self.repo.get_definition(key) is not None:
|
|
||||||
raise AppError(409, "CONFLICT", "attribute definition already exists", {"key": key})
|
|
||||||
return self.repo.add_definition(row)
|
|
||||||
|
|
||||||
def create_profile(self, payload: dict[str, Any]) -> orm.ProductProfileRecord:
|
|
||||||
try:
|
|
||||||
model_versions = self.repo.scoring_versions()
|
|
||||||
scoring_version = payload.get("scoring_version")
|
|
||||||
model = self.repo.get_scoring_model(scoring_version) if scoring_version else None
|
|
||||||
if model is None:
|
|
||||||
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version does not exist")
|
|
||||||
profile = parse_profile(
|
|
||||||
payload,
|
|
||||||
attribute_definitions=self.repo.all_definitions(),
|
|
||||||
scoring_versions=model_versions,
|
|
||||||
supported_strategies=set(model.supported_preference_strategies),
|
|
||||||
)
|
|
||||||
except DomainValidationError as exc:
|
|
||||||
raise AppError(422, "PROFILE_INVALID", str(exc), exc.details) from exc
|
|
||||||
except AppError:
|
|
||||||
raise
|
|
||||||
row = orm.ProductProfileRecord(
|
|
||||||
id=profile.id,
|
|
||||||
version=profile.version,
|
|
||||||
name=profile.name,
|
|
||||||
category=profile.category,
|
|
||||||
scoring_version=profile.scoring_version,
|
|
||||||
definition=profile.definition,
|
|
||||||
definition_hash=profile.definition_hash,
|
|
||||||
created_at=_now(),
|
|
||||||
)
|
|
||||||
if self.repo.get_profile(profile.id, profile.version) is not None:
|
|
||||||
raise AppError(409, "CONFLICT", "profile version already exists", {"id": profile.id, "version": profile.version})
|
|
||||||
return self.repo.add_profile(row)
|
|
||||||
|
|
||||||
def create_product(self, payload: dict[str, Any]) -> Product:
|
|
||||||
product_id = _uuid(payload.get("id"))
|
|
||||||
if self.repo.get_product(product_id) is not None:
|
|
||||||
raise AppError(409, "CONFLICT", "product already exists", {"id": str(product_id)})
|
|
||||||
row = orm.ProductRecord(
|
|
||||||
id=product_id,
|
|
||||||
category=str(payload["category"]),
|
|
||||||
brand=str(payload["brand"]),
|
|
||||||
name=str(payload["name"]),
|
|
||||||
model=payload.get("model"),
|
|
||||||
created_at=_now(),
|
|
||||||
)
|
|
||||||
return self.repo.add_product(row)
|
|
||||||
|
|
||||||
def create_variant(self, product_id: UUID, payload: dict[str, Any]) -> ProductVariant:
|
|
||||||
if self.repo.get_product(product_id) is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
|
|
||||||
variant_id = _uuid(payload.get("id"))
|
|
||||||
if self.repo.get_variant(variant_id) is not None:
|
|
||||||
raise AppError(409, "CONFLICT", "variant already exists", {"id": str(variant_id)})
|
|
||||||
row = orm.ProductVariantRecord(
|
|
||||||
id=variant_id,
|
|
||||||
product_id=product_id,
|
|
||||||
name=str(payload["name"]),
|
|
||||||
manufacturer_sku=payload.get("manufacturer_sku"),
|
|
||||||
identifiers=payload.get("identifiers") or {},
|
|
||||||
created_at=_now(),
|
|
||||||
)
|
|
||||||
return self.repo.add_variant(row)
|
|
||||||
|
|
||||||
def add_attribute(self, variant_id: UUID, payload: dict[str, Any]) -> AttributeStatement:
|
|
||||||
variant = self.repo.get_variant(variant_id)
|
|
||||||
if variant is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
|
||||||
definition = self.repo.get_definition(payload["attribute"])
|
|
||||||
if definition is None:
|
|
||||||
raise AppError(422, "UNKNOWN_ATTRIBUTE", "attribute definition does not exist")
|
|
||||||
if self.repo.get_source(payload["source_id"]) is None:
|
|
||||||
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
|
|
||||||
observed_at = require_utc(payload["observed_at"], "observed_at")
|
|
||||||
ingested_at = _now()
|
|
||||||
if observed_at > ingested_at:
|
|
||||||
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
|
|
||||||
confidence = Decimal(str(payload["confidence"]))
|
|
||||||
if not Decimal("0") <= confidence <= Decimal("1"):
|
|
||||||
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
|
|
||||||
statement = AttributeStatement(
|
|
||||||
id=_uuid(payload.get("id")),
|
|
||||||
variant_id=variant_id,
|
|
||||||
attribute=definition.key,
|
|
||||||
raw_value=payload["raw_value"],
|
|
||||||
normalized_value=payload.get("normalized_value"),
|
|
||||||
unit=payload.get("unit"),
|
|
||||||
normalization_status=NormalizationStatus(payload["normalization_status"]),
|
|
||||||
normalizer_version=payload.get("normalizer_version") or self.normalizer.version,
|
|
||||||
confidence=confidence,
|
|
||||||
source_id=payload["source_id"],
|
|
||||||
source_reference=payload["source_reference"],
|
|
||||||
observed_at=observed_at,
|
|
||||||
ingested_at=ingested_at,
|
|
||||||
normalization_error=payload.get("normalization_error"),
|
|
||||||
)
|
|
||||||
statement = self.normalizer.validate(definition, statement)
|
|
||||||
if self.repo.get_attribute(statement.id) is not None:
|
|
||||||
raise AppError(409, "CONFLICT", "attribute statement already exists", {"id": str(statement.id)})
|
|
||||||
row = orm.VariantAttributeRecord(
|
|
||||||
id=statement.id,
|
|
||||||
variant_id=variant_id,
|
|
||||||
attribute_key=statement.attribute,
|
|
||||||
source_id=statement.source_id,
|
|
||||||
raw_value=statement.raw_value,
|
|
||||||
normalized_value=statement.normalized_value,
|
|
||||||
unit=statement.unit,
|
|
||||||
normalization_status=statement.normalization_status.value,
|
|
||||||
normalizer_version=statement.normalizer_version,
|
|
||||||
confidence=statement.confidence,
|
|
||||||
source_reference=statement.source_reference,
|
|
||||||
observed_at=statement.observed_at,
|
|
||||||
ingested_at=statement.ingested_at,
|
|
||||||
normalization_error=statement.normalization_error,
|
|
||||||
)
|
|
||||||
return self.repo.add_attribute(row)
|
|
||||||
|
|
||||||
def create_offer(self, variant_id: UUID, payload: dict[str, Any]) -> Offer:
|
|
||||||
if self.repo.get_variant(variant_id) is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
|
||||||
if self.repo.get_source(payload["source_id"]) is None:
|
|
||||||
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
|
|
||||||
offer_id = _uuid(payload.get("id"))
|
|
||||||
if self.repo.get_offer(offer_id) is not None:
|
|
||||||
raise AppError(409, "CONFLICT", "offer already exists", {"id": str(offer_id)})
|
|
||||||
if self.repo.get_offer_by_external(payload["source_id"], str(payload["external_id"])) is not None:
|
|
||||||
raise AppError(409, "CONFLICT", "offer source/external_id already exists")
|
|
||||||
row = orm.OfferRecord(
|
|
||||||
id=offer_id,
|
|
||||||
variant_id=variant_id,
|
|
||||||
source_id=payload["source_id"],
|
|
||||||
external_id=str(payload["external_id"]),
|
|
||||||
seller=str(payload["seller"]),
|
|
||||||
url=payload.get("url"),
|
|
||||||
market=str(payload["market"]),
|
|
||||||
currency=validate_currency(str(payload["currency"])),
|
|
||||||
condition=OfferCondition(payload["condition"]).value,
|
|
||||||
created_at=_now(),
|
|
||||||
)
|
|
||||||
return self.repo.add_offer(row)
|
|
||||||
|
|
||||||
def add_price(self, offer_id: UUID, payload: dict[str, Any]) -> tuple[PriceObservation, bool]:
|
|
||||||
offer = self.repo.get_offer(offer_id)
|
|
||||||
if offer is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
|
||||||
if payload["source_id"] != offer.source_id:
|
|
||||||
raise AppError(422, "SOURCE_MISMATCH", "source_id must match the offer")
|
|
||||||
currency = validate_currency(str(payload["currency"]))
|
|
||||||
if currency != offer.currency:
|
|
||||||
raise AppError(422, "CURRENCY_MISMATCH", "currency must match the offer")
|
|
||||||
observed_at = require_utc(payload["observed_at"], "observed_at")
|
|
||||||
ingested_at = _now()
|
|
||||||
if observed_at > ingested_at:
|
|
||||||
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
|
|
||||||
item_price = Decimal(str(payload["item_price"]))
|
|
||||||
if item_price <= 0:
|
|
||||||
raise AppError(422, "ITEM_PRICE", "item_price must be > 0")
|
|
||||||
shipping = payload.get("shipping_price")
|
|
||||||
shipping_price = None if shipping is None else Decimal(str(shipping))
|
|
||||||
if shipping_price is not None and shipping_price < 0:
|
|
||||||
raise AppError(422, "SHIPPING_PRICE", "shipping_price must be >= 0")
|
|
||||||
confidence = Decimal(str(payload["confidence"]))
|
|
||||||
if not Decimal("0") <= confidence <= Decimal("1"):
|
|
||||||
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
|
|
||||||
external_id = str(payload["external_observation_id"])
|
|
||||||
existing = self.repo.get_observation_by_external(offer_id, external_id)
|
|
||||||
incoming = {
|
|
||||||
"source_id": payload["source_id"],
|
|
||||||
"observed_at": observed_at,
|
|
||||||
"item_price": item_price,
|
|
||||||
"shipping_price": shipping_price,
|
|
||||||
"currency": currency,
|
|
||||||
"availability": Availability(payload["availability"]).value,
|
|
||||||
"raw_payload": payload["raw_payload"],
|
|
||||||
"confidence": confidence,
|
|
||||||
}
|
|
||||||
if existing is not None:
|
|
||||||
if _observation_content(existing) != incoming:
|
|
||||||
raise AppError(409, "CONFLICT", "price observation content differs")
|
|
||||||
return to_observation(existing), False
|
|
||||||
row = orm.PriceObservationRecord(
|
|
||||||
id=_uuid(payload.get("id")),
|
|
||||||
offer_id=offer_id,
|
|
||||||
source_id=payload["source_id"],
|
|
||||||
external_observation_id=external_id,
|
|
||||||
observed_at=observed_at,
|
|
||||||
ingested_at=ingested_at,
|
|
||||||
item_price=item_price,
|
|
||||||
shipping_price=shipping_price,
|
|
||||||
currency=currency,
|
|
||||||
availability=incoming["availability"],
|
|
||||||
raw_payload=payload["raw_payload"],
|
|
||||||
confidence=confidence,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
return self.repo.add_observation(row), True
|
|
||||||
except IntegrityError:
|
|
||||||
self.session.rollback()
|
|
||||||
existing = self.repo.get_observation_by_external(offer_id, external_id)
|
|
||||||
if existing is None:
|
|
||||||
raise
|
|
||||||
if _observation_content(existing) != incoming:
|
|
||||||
raise AppError(409, "CONFLICT", "price observation content differs")
|
|
||||||
return to_observation(existing), False
|
|
||||||
|
|
||||||
|
|
||||||
def _observation_content(row: orm.PriceObservationRecord) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"source_id": row.source_id,
|
|
||||||
"observed_at": row.observed_at,
|
|
||||||
"item_price": Decimal(str(row.item_price)),
|
|
||||||
"shipping_price": None if row.shipping_price is None else Decimal(str(row.shipping_price)),
|
|
||||||
"currency": row.currency,
|
|
||||||
"availability": row.availability,
|
|
||||||
"raw_payload": row.raw_payload,
|
|
||||||
"confidence": Decimal(str(row.confidence)),
|
|
||||||
}
|
|
||||||
|
|
@ -1,346 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID, uuid4
|
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.clock import now_utc
|
|
||||||
from app.discovery.criteria import criterion_payload, parse_project_criteria
|
|
||||||
from app.discovery.definitions import get_definition, get_definition_by_key, upsert_definition
|
|
||||||
from app.discovery.enums import CandidateStatus, DiffStatus, ProjectStatus, RankingStatus, SearchStatus
|
|
||||||
from app.domain.serialize import jsonable
|
|
||||||
from app.domain.validation import DomainValidationError
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.persistence.discovery_models import (
|
|
||||||
BuyingProjectCriteriaRecord,
|
|
||||||
BuyingProjectRecord,
|
|
||||||
DiscoverySearchRecord,
|
|
||||||
ProductCategoryRecord,
|
|
||||||
ProjectCandidateRecord,
|
|
||||||
ProjectCriterionRecord,
|
|
||||||
SearchRunItemRecord,
|
|
||||||
SearchRunRecord,
|
|
||||||
)
|
|
||||||
from app.persistence.repositories import CatalogRepository
|
|
||||||
from app.services.catalog_service import CatalogService, ensure_product_category
|
|
||||||
|
|
||||||
|
|
||||||
INACTIVE_SEARCH = {SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}
|
|
||||||
|
|
||||||
|
|
||||||
def _uuid(value: UUID | None) -> UUID:
|
|
||||||
return value or uuid4()
|
|
||||||
|
|
||||||
|
|
||||||
class ProjectService:
|
|
||||||
def __init__(self, session: Session) -> None:
|
|
||||||
self.session = session
|
|
||||||
self.catalog = CatalogRepository(session)
|
|
||||||
self.categories = CatalogService(session)
|
|
||||||
|
|
||||||
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
template_id = payload.get("template_id") or payload.get("profile_id")
|
|
||||||
template_version = payload.get("template_version") or payload.get("profile_version")
|
|
||||||
profile = None
|
|
||||||
if template_id is not None:
|
|
||||||
if template_version is None:
|
|
||||||
raise AppError(422, "VALIDATION_ERROR", "template_version is required when template_id is set")
|
|
||||||
profile = self.catalog.get_profile(str(template_id), int(template_version))
|
|
||||||
if profile is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "template version not found")
|
|
||||||
product_category_id = payload.get("product_category_id") or (None if profile is None else profile.category)
|
|
||||||
if not product_category_id:
|
|
||||||
raise AppError(422, "VALIDATION_ERROR", "product_category_id is required")
|
|
||||||
if self.session.get(ProductCategoryRecord, product_category_id) is None:
|
|
||||||
ensure_product_category(self.session, str(product_category_id))
|
|
||||||
market = payload.get("market") or (None if profile is None else profile.market)
|
|
||||||
currency = payload.get("currency") or (None if profile is None else profile.currency)
|
|
||||||
if not market or not currency:
|
|
||||||
raise AppError(422, "VALIDATION_ERROR", "market and currency are required")
|
|
||||||
target_budget = payload.get("target_budget")
|
|
||||||
if target_budget is None and profile is not None and getattr(profile, "budget", None) is not None:
|
|
||||||
target_budget = getattr(profile.budget, "target", None)
|
|
||||||
criteria_payload = payload.get("criteria")
|
|
||||||
if not criteria_payload:
|
|
||||||
raise AppError(422, "CRITERIA_REQUIRED", "criteria are required")
|
|
||||||
try:
|
|
||||||
parsed = parse_project_criteria(criteria_payload)
|
|
||||||
except DomainValidationError as exc:
|
|
||||||
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
|
|
||||||
shortlist_size = int(payload.get("shortlist_size") or 5)
|
|
||||||
if shortlist_size < 1:
|
|
||||||
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
|
|
||||||
now = now_utc()
|
|
||||||
project = BuyingProjectRecord(
|
|
||||||
id=_uuid(payload.get("id")),
|
|
||||||
name=str(payload["name"]),
|
|
||||||
status=ProjectStatus.ACTIVE.value,
|
|
||||||
workspace_id=payload.get("workspace_id"),
|
|
||||||
product_category_id=str(product_category_id),
|
|
||||||
market=str(market),
|
|
||||||
currency=str(currency),
|
|
||||||
target_budget=None if target_budget is None else Decimal(str(target_budget)),
|
|
||||||
template_id=None if profile is None else profile.id,
|
|
||||||
template_version=None if profile is None else profile.version,
|
|
||||||
shortlist_size=shortlist_size,
|
|
||||||
criteria_version=1,
|
|
||||||
created_at=now,
|
|
||||||
updated_at=now,
|
|
||||||
archived_at=None,
|
|
||||||
)
|
|
||||||
self.session.add(project)
|
|
||||||
self.session.flush()
|
|
||||||
self._store_criteria(project.id, 1, parsed, str(product_category_id), now)
|
|
||||||
return self.get(project.id)
|
|
||||||
|
|
||||||
def replace_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
project = self._require_project(project_id)
|
|
||||||
try:
|
|
||||||
parsed = parse_project_criteria(payload)
|
|
||||||
except DomainValidationError as exc:
|
|
||||||
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
|
|
||||||
current = self.latest_criteria(project_id)
|
|
||||||
version = (current.version if current is not None else 0) + 1
|
|
||||||
now = now_utc()
|
|
||||||
self._store_criteria(project.id, version, parsed, project.product_category_id, now)
|
|
||||||
project.criteria_version = version
|
|
||||||
project.updated_at = now
|
|
||||||
for search in self.session.scalars(
|
|
||||||
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
|
|
||||||
):
|
|
||||||
if search.status not in INACTIVE_SEARCH:
|
|
||||||
search.status = SearchStatus.OUT_OF_SYNC.value
|
|
||||||
search.updated_at = now
|
|
||||||
self.session.flush()
|
|
||||||
return self.get(project.id)
|
|
||||||
|
|
||||||
def add_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
return self.replace_criteria(project_id, payload)
|
|
||||||
|
|
||||||
def patch(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
project = self._require_project(project_id)
|
|
||||||
allowed = {"name", "target_budget", "shortlist_size", "status"}
|
|
||||||
unknown = set(payload) - allowed
|
|
||||||
if unknown:
|
|
||||||
raise AppError(422, "VALIDATION_ERROR", "field cannot be patched", {"fields": sorted(unknown)})
|
|
||||||
if "name" in payload:
|
|
||||||
project.name = str(payload["name"])
|
|
||||||
if "target_budget" in payload:
|
|
||||||
project.target_budget = None if payload["target_budget"] is None else Decimal(str(payload["target_budget"]))
|
|
||||||
if "shortlist_size" in payload:
|
|
||||||
size = int(payload["shortlist_size"])
|
|
||||||
if size < 1:
|
|
||||||
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
|
|
||||||
project.shortlist_size = size
|
|
||||||
if "status" in payload:
|
|
||||||
status = str(payload["status"])
|
|
||||||
if status not in {item.value for item in ProjectStatus}:
|
|
||||||
raise AppError(422, "VALIDATION_ERROR", "invalid project status")
|
|
||||||
project.status = status
|
|
||||||
if status == ProjectStatus.ARCHIVED.value:
|
|
||||||
project.archived_at = now_utc()
|
|
||||||
project.updated_at = now_utc()
|
|
||||||
self.session.flush()
|
|
||||||
return self.get(project.id)
|
|
||||||
|
|
||||||
def get(self, project_id: UUID) -> dict[str, Any]:
|
|
||||||
project = self._require_project(project_id)
|
|
||||||
criteria = self.latest_criteria(project_id)
|
|
||||||
return self._out(project, criteria)
|
|
||||||
|
|
||||||
def get_criteria(self, project_id: UUID) -> dict[str, Any]:
|
|
||||||
project = self._require_project(project_id)
|
|
||||||
criteria = self.latest_criteria(project_id)
|
|
||||||
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
|
|
||||||
return jsonable(
|
|
||||||
{
|
|
||||||
"project_id": project.id,
|
|
||||||
"criteria_version": None if criteria is None else criteria.version,
|
|
||||||
"items": items,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def list(self, limit: int, offset: int) -> tuple[list[dict[str, Any]], int]:
|
|
||||||
total = self.session.scalar(select(func.count()).select_from(BuyingProjectRecord)) or 0
|
|
||||||
rows = self.session.scalars(
|
|
||||||
select(BuyingProjectRecord).order_by(BuyingProjectRecord.created_at).limit(limit).offset(offset)
|
|
||||||
).all()
|
|
||||||
return [self._summary(row) for row in rows], total
|
|
||||||
|
|
||||||
def latest_criteria(self, project_id: UUID) -> BuyingProjectCriteriaRecord | None:
|
|
||||||
return self.session.scalars(
|
|
||||||
select(BuyingProjectCriteriaRecord)
|
|
||||||
.where(BuyingProjectCriteriaRecord.project_id == project_id)
|
|
||||||
.order_by(BuyingProjectCriteriaRecord.version.desc())
|
|
||||||
).first()
|
|
||||||
|
|
||||||
def criteria_version(self, project_id: UUID, version: int) -> BuyingProjectCriteriaRecord | None:
|
|
||||||
return self.session.get(BuyingProjectCriteriaRecord, (project_id, version))
|
|
||||||
|
|
||||||
def criteria_items(self, project_id: UUID, version: int) -> list[dict[str, Any]]:
|
|
||||||
rows = self.session.scalars(
|
|
||||||
select(ProjectCriterionRecord)
|
|
||||||
.where(ProjectCriterionRecord.project_id == project_id, ProjectCriterionRecord.version == version)
|
|
||||||
.order_by(ProjectCriterionRecord.item_id)
|
|
||||||
).all()
|
|
||||||
return [criterion_payload(row) for row in rows]
|
|
||||||
|
|
||||||
def available_criteria(self, project_id: UUID) -> dict[str, Any]:
|
|
||||||
project = self._require_project(project_id)
|
|
||||||
sources = self.categories.sources_for_category(project.product_category_id)
|
|
||||||
source = sources[0]["source"] if sources else "geizhals"
|
|
||||||
return self.categories.available_criteria(project.product_category_id, source)
|
|
||||||
|
|
||||||
def _store_criteria(
|
|
||||||
self,
|
|
||||||
project_id: UUID,
|
|
||||||
version: int,
|
|
||||||
parsed: dict[str, Any],
|
|
||||||
category: str | None,
|
|
||||||
created_at,
|
|
||||||
) -> None:
|
|
||||||
self.session.add(
|
|
||||||
BuyingProjectCriteriaRecord(
|
|
||||||
project_id=project_id,
|
|
||||||
version=version,
|
|
||||||
definition_hash=parsed["definition_hash"],
|
|
||||||
created_at=created_at,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
self.session.flush()
|
|
||||||
for item in parsed["items"]:
|
|
||||||
definition = self._resolve_definition(item, category)
|
|
||||||
self.session.add(
|
|
||||||
ProjectCriterionRecord(
|
|
||||||
id=uuid4(),
|
|
||||||
project_id=project_id,
|
|
||||||
version=version,
|
|
||||||
item_id=item["id"],
|
|
||||||
criterion_definition_id=definition.id,
|
|
||||||
criterion_key=definition.key,
|
|
||||||
operator=item["operator"],
|
|
||||||
value=item["value"],
|
|
||||||
enabled_for_discovery=item["enabled_for_discovery"],
|
|
||||||
created_at=created_at,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
self.session.flush()
|
|
||||||
|
|
||||||
def _resolve_definition(self, item: dict[str, Any], category: str | None):
|
|
||||||
if item.get("criterion_definition_id") is not None:
|
|
||||||
definition = get_definition(self.session, item["criterion_definition_id"])
|
|
||||||
if definition is None:
|
|
||||||
raise AppError(422, "UNKNOWN_CRITERION", "criterion definition not found")
|
|
||||||
return definition
|
|
||||||
existing = get_definition_by_key(self.session, str(item["criterion"]))
|
|
||||||
if existing is not None:
|
|
||||||
return existing
|
|
||||||
if not item.get("value_type"):
|
|
||||||
raise AppError(
|
|
||||||
422,
|
|
||||||
"UNKNOWN_CRITERION",
|
|
||||||
"criterion definition missing; provide value_type or create it first",
|
|
||||||
{"criterion": item["criterion"]},
|
|
||||||
)
|
|
||||||
return upsert_definition(
|
|
||||||
self.session,
|
|
||||||
{
|
|
||||||
"key": item["criterion"],
|
|
||||||
"origin": "INTERNAL",
|
|
||||||
"product_category_id": item.get("category") or category,
|
|
||||||
"label": item["criterion"],
|
|
||||||
"value_type": item["value_type"],
|
|
||||||
"unit": item.get("unit"),
|
|
||||||
"semantic": item.get("semantic"),
|
|
||||||
"category": item.get("category") or category,
|
|
||||||
"scoring_attribute_key": item["criterion"],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
def _require_project(self, project_id: UUID) -> BuyingProjectRecord:
|
|
||||||
project = self.session.get(BuyingProjectRecord, project_id)
|
|
||||||
if project is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "buying project not found")
|
|
||||||
return project
|
|
||||||
|
|
||||||
def _out(self, project: BuyingProjectRecord, criteria: BuyingProjectCriteriaRecord | None) -> dict[str, Any]:
|
|
||||||
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
|
|
||||||
return jsonable(
|
|
||||||
{
|
|
||||||
"id": project.id,
|
|
||||||
"name": project.name,
|
|
||||||
"status": project.status,
|
|
||||||
"workspace_id": project.workspace_id,
|
|
||||||
"product_category_id": project.product_category_id,
|
|
||||||
"market": project.market,
|
|
||||||
"currency": project.currency,
|
|
||||||
"target_budget": project.target_budget,
|
|
||||||
"template_id": project.template_id,
|
|
||||||
"template_version": project.template_version,
|
|
||||||
"profile_id": project.template_id,
|
|
||||||
"profile_version": project.template_version,
|
|
||||||
"shortlist_size": project.shortlist_size,
|
|
||||||
"created_at": project.created_at,
|
|
||||||
"updated_at": project.updated_at,
|
|
||||||
"archived_at": project.archived_at,
|
|
||||||
"criteria_version": project.criteria_version if criteria is None else criteria.version,
|
|
||||||
"criteria": None if criteria is None else {"items": items},
|
|
||||||
"criteria_hash": None if criteria is None else criteria.definition_hash,
|
|
||||||
"ranking_status": RankingStatus.NOT_CONFIGURED.value
|
|
||||||
if project.template_id is None
|
|
||||||
else RankingStatus.CONFIGURED.value,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def _summary(self, project: BuyingProjectRecord) -> dict[str, Any]:
|
|
||||||
category = self.session.get(ProductCategoryRecord, project.product_category_id)
|
|
||||||
searches = list(
|
|
||||||
self.session.scalars(
|
|
||||||
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
active = next((row for row in searches if row.status not in INACTIVE_SEARCH), searches[0] if searches else None)
|
|
||||||
last_run = None
|
|
||||||
if active is not None:
|
|
||||||
last_run = self.session.scalars(
|
|
||||||
select(SearchRunRecord)
|
|
||||||
.where(SearchRunRecord.discovery_search_id == active.id)
|
|
||||||
.order_by(SearchRunRecord.started_at.desc())
|
|
||||||
).first()
|
|
||||||
candidates = list(
|
|
||||||
self.session.scalars(
|
|
||||||
select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
counts = {"NEW": 0, "CHANGED": 0, "MISSING": 0}
|
|
||||||
if last_run is not None:
|
|
||||||
for item in self.session.scalars(
|
|
||||||
select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == last_run.id)
|
|
||||||
):
|
|
||||||
if item.diff_status in counts:
|
|
||||||
counts[item.diff_status] += 1
|
|
||||||
return jsonable(
|
|
||||||
{
|
|
||||||
"id": project.id,
|
|
||||||
"name": project.name,
|
|
||||||
"product_category": None
|
|
||||||
if category is None
|
|
||||||
else {"id": category.id, "name": category.name, "status": category.status},
|
|
||||||
"status": project.status,
|
|
||||||
"last_successful_run_at": None if active is None else active.last_successful_run_at,
|
|
||||||
"next_run_at": None if active is None else active.next_run_at,
|
|
||||||
"discovery_status": None if active is None else active.status,
|
|
||||||
"candidate_count": len(candidates),
|
|
||||||
"shortlist_count": sum(1 for row in candidates if row.status == CandidateStatus.SHORTLISTED.value),
|
|
||||||
"new_count": last_run.new_count if last_run is not None else counts[DiffStatus.NEW.value],
|
|
||||||
"changed_count": last_run.changed_count if last_run is not None else counts[DiffStatus.CHANGED.value],
|
|
||||||
"missing_count": last_run.missing_count if last_run is not None else counts[DiffStatus.MISSING.value],
|
|
||||||
"current_search_result_count": None if last_run is None else last_run.result_count,
|
|
||||||
"ranking_status": RankingStatus.NOT_CONFIGURED.value
|
|
||||||
if project.template_id is None
|
|
||||||
else RankingStatus.CONFIGURED.value,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
@ -1,85 +0,0 @@
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any, Protocol
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class SourceProduct:
|
|
||||||
source_key: str
|
|
||||||
external_id: str
|
|
||||||
id: UUID
|
|
||||||
category: str
|
|
||||||
brand: str
|
|
||||||
name: str
|
|
||||||
model: str | None
|
|
||||||
variants: tuple["SourceVariant", ...]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class SourceVariant:
|
|
||||||
id: UUID
|
|
||||||
name: str
|
|
||||||
manufacturer_sku: str | None
|
|
||||||
identifiers: dict[str, Any]
|
|
||||||
attributes: tuple["SourceAttribute", ...]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class SourceAttribute:
|
|
||||||
id: UUID
|
|
||||||
attribute: str
|
|
||||||
raw_value: Any
|
|
||||||
normalized_value: Any
|
|
||||||
unit: str | None
|
|
||||||
normalization_status: str
|
|
||||||
normalizer_version: str
|
|
||||||
confidence: Decimal
|
|
||||||
source_reference: str
|
|
||||||
observed_at: datetime
|
|
||||||
normalization_error: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class SourceOffer:
|
|
||||||
id: UUID
|
|
||||||
variant_id: UUID
|
|
||||||
external_id: str
|
|
||||||
seller: str
|
|
||||||
url: str | None
|
|
||||||
market: str
|
|
||||||
currency: str
|
|
||||||
condition: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class SourcePriceObservation:
|
|
||||||
id: UUID | None
|
|
||||||
external_offer_id: str
|
|
||||||
external_observation_id: str
|
|
||||||
observed_at: datetime
|
|
||||||
item_price: Decimal
|
|
||||||
shipping_price: Decimal | None
|
|
||||||
currency: str
|
|
||||||
availability: str
|
|
||||||
raw_payload: Any
|
|
||||||
confidence: Decimal
|
|
||||||
|
|
||||||
|
|
||||||
class ProductSource(Protocol):
|
|
||||||
source_key: str
|
|
||||||
|
|
||||||
def get_product(self, external_id: str) -> SourceProduct: ...
|
|
||||||
def get_offers(self, external_product_id: str) -> list[SourceOffer]: ...
|
|
||||||
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]: ...
|
|
||||||
|
|
||||||
|
|
||||||
class DiscoverySourceAdapter(Protocol):
|
|
||||||
source_key: str
|
|
||||||
|
|
||||||
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]: ...
|
|
||||||
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: ...
|
|
||||||
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]: ...
|
|
||||||
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]: ...
|
|
||||||
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]: ...
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
from app.sources.geizhals.adapter import GeizhalsAdapter
|
|
||||||
|
|
||||||
__all__ = ["GeizhalsAdapter"]
|
|
||||||
|
|
@ -1,130 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import parse_qs, urlparse
|
|
||||||
|
|
||||||
from app.discovery.enums import FilterType
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.settings import get_settings
|
|
||||||
from app.sources.geizhals.category_url import parse_category_key, with_extended_filters
|
|
||||||
from app.sources.geizhals.live_page import load_category_page, load_result_page
|
|
||||||
from app.sources.geizhals.result_parser import parse_result_html, with_page
|
|
||||||
from app.sources.geizhals.taxonomy_api import fetch_taxonomy_api
|
|
||||||
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
|
|
||||||
|
|
||||||
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
|
||||||
|
|
||||||
|
|
||||||
class GeizhalsAdapter:
|
|
||||||
source_key = "geizhals"
|
|
||||||
|
|
||||||
def __init__(self, fixture_dir: Path | None = None, live: bool | None = None) -> None:
|
|
||||||
self.fixture_dir = fixture_dir or FIXTURE_DIR
|
|
||||||
settings = get_settings()
|
|
||||||
self.live = (not settings.geizhals_use_fixtures) if live is None else live
|
|
||||||
|
|
||||||
def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]:
|
|
||||||
if not self.live:
|
|
||||||
return self._load(f"{category_ref}-taxonomy.json")
|
|
||||||
url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}")
|
|
||||||
category_ref = parse_category_key(url)
|
|
||||||
api_payload = fetch_taxonomy_api(source_url or url, category_ref=category_ref)
|
|
||||||
if api_payload and api_payload.get("filters"):
|
|
||||||
return api_payload
|
|
||||||
html, mode, status = load_category_page(
|
|
||||||
url,
|
|
||||||
category_ref=category_ref,
|
|
||||||
playwright=get_settings().geizhals_playwright,
|
|
||||||
)
|
|
||||||
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url)
|
|
||||||
if payload.get("blocked") or not payload.get("filters"):
|
|
||||||
raise AppError(
|
|
||||||
422,
|
|
||||||
"TAXONOMY_EMPTY",
|
|
||||||
"Geizhals-Taxonomie konnte nicht aus der Kategorieseite gelesen werden",
|
|
||||||
{"url": url, "http_status": status, "mode": mode},
|
|
||||||
)
|
|
||||||
payload["discovery_mode"] = mode
|
|
||||||
payload["http_status"] = status
|
|
||||||
payload["taxonomy_status"] = "DEGRADED" if mode == "recorded_html" else "READY"
|
|
||||||
return payload
|
|
||||||
|
|
||||||
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
|
|
||||||
from app.discovery.urlgen import generate_search_url
|
|
||||||
|
|
||||||
category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None)
|
|
||||||
category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
|
|
||||||
url, internal_only = generate_search_url(
|
|
||||||
category_url=category_url,
|
|
||||||
criteria_items=criteria,
|
|
||||||
mappings=mappings,
|
|
||||||
)
|
|
||||||
return {"url": url, "internal_only": internal_only}
|
|
||||||
|
|
||||||
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
page = int(pagination.get("page") or 1)
|
|
||||||
if not self.live:
|
|
||||||
category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0]
|
|
||||||
return self._load(f"{category}-search-page{page}.json")
|
|
||||||
url = with_page(str(query["url"]), page)
|
|
||||||
html, mode, status, blocked = load_result_page(url, playwright=get_settings().geizhals_playwright)
|
|
||||||
if blocked:
|
|
||||||
return {
|
|
||||||
"page": page,
|
|
||||||
"page_count": page,
|
|
||||||
"result_count": 0,
|
|
||||||
"http_status": status or 403,
|
|
||||||
"blocked": True,
|
|
||||||
"challenge": "captcha" if status in {401, 403} else None,
|
|
||||||
"items": [],
|
|
||||||
"discovery_mode": mode,
|
|
||||||
}
|
|
||||||
payload = parse_result_html(html, page_url=url)
|
|
||||||
payload["page"] = page
|
|
||||||
payload["http_status"] = status
|
|
||||||
payload["discovery_mode"] = mode
|
|
||||||
return payload
|
|
||||||
|
|
||||||
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]:
|
|
||||||
if response.get("html") and not response.get("items"):
|
|
||||||
return list(parse_result_html(str(response["html"]), page_url=str(response.get("url") or "")).get("items") or [])
|
|
||||||
return list(response.get("items") or [])
|
|
||||||
|
|
||||||
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
attrs = result.get("normalized_attributes") or {}
|
|
||||||
return {
|
|
||||||
"source_external_id": result.get("source_external_id"),
|
|
||||||
"source_url": result.get("source_url"),
|
|
||||||
"source_title": result.get("source_title"),
|
|
||||||
"manufacturer": result.get("manufacturer"),
|
|
||||||
"model": result.get("model"),
|
|
||||||
"manufacturer_sku": result.get("manufacturer_sku"),
|
|
||||||
"gtin_ean": result.get("gtin_ean"),
|
|
||||||
"best_price": result.get("best_price"),
|
|
||||||
"currency": result.get("currency") or "EUR",
|
|
||||||
"availability": result.get("availability") or "unknown",
|
|
||||||
"seller_count": result.get("seller_count"),
|
|
||||||
"raw_attributes": result.get("raw_attributes") or {},
|
|
||||||
"normalized_attributes": attrs,
|
|
||||||
}
|
|
||||||
|
|
||||||
def _load(self, name: str) -> dict[str, Any]:
|
|
||||||
path = self.fixture_dir / name
|
|
||||||
if not path.exists():
|
|
||||||
raise AppError(404, "FIXTURE_NOT_FOUND", f"geizhals fixture missing: {name}")
|
|
||||||
return json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
def _category(self, mappings: list[dict[str, Any]], criteria: list[dict[str, Any]]) -> str:
|
|
||||||
for mapping in mappings:
|
|
||||||
if mapping.get("category_key"):
|
|
||||||
return str(mapping["category_key"])
|
|
||||||
return "nb"
|
|
||||||
|
|
||||||
|
|
||||||
def filter_type(raw: str) -> str:
|
|
||||||
try:
|
|
||||||
return FilterType(raw).value
|
|
||||||
except ValueError:
|
|
||||||
return FilterType.UNKNOWN.value
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
|
|
||||||
|
|
||||||
from app.errors import AppError
|
|
||||||
|
|
||||||
CATEGORY_PATH_RE = re.compile(r"/cat/([^/?#]+)", re.IGNORECASE)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_category_key(source_url: str) -> str:
|
|
||||||
parts = urlsplit(source_url.strip())
|
|
||||||
query = parse_qs(parts.query)
|
|
||||||
if query.get("cat"):
|
|
||||||
return str(query["cat"][0])
|
|
||||||
match = CATEGORY_PATH_RE.search(parts.path)
|
|
||||||
if match:
|
|
||||||
return match.group(1)
|
|
||||||
raise AppError(422, "SOURCE_URL_INVALID", "Quellpfad muss eine Geizhals-Kategorie enthalten (cat=…)")
|
|
||||||
|
|
||||||
|
|
||||||
def slugify(value: str) -> str:
|
|
||||||
text = value.strip().casefold().translate(str.maketrans("äöüß", "aous"))
|
|
||||||
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
|
|
||||||
return text or "category"
|
|
||||||
|
|
||||||
|
|
||||||
def with_extended_filters(source_url: str) -> str:
|
|
||||||
parts = urlsplit(source_url.strip())
|
|
||||||
query = [(name, value) for name, value in _query_items(parts.query) if name != "promode"]
|
|
||||||
query.append(("promode", "true"))
|
|
||||||
return urlunsplit((parts.scheme or "https", parts.netloc, parts.path or "/", urlencode(query), parts.fragment))
|
|
||||||
|
|
||||||
|
|
||||||
def _query_items(query: str) -> list[tuple[str, str]]:
|
|
||||||
from urllib.parse import parse_qsl
|
|
||||||
|
|
||||||
return parse_qsl(query, keep_blank_values=True)
|
|
||||||
|
|
@ -1,139 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.sources.geizhals.taxonomy_parser import is_challenge_html
|
|
||||||
|
|
||||||
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
|
||||||
BROWSER_HEADERS = {
|
|
||||||
"User-Agent": (
|
|
||||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
||||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
|
|
||||||
),
|
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
||||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
|
||||||
}
|
|
||||||
EXPAND_LABELS = (
|
|
||||||
"Erweiterte Filter",
|
|
||||||
"Alle Filter ausklappen",
|
|
||||||
"Immer alle Filter anzeigen",
|
|
||||||
"weitere Filter anzeigen",
|
|
||||||
)
|
|
||||||
MORE_OPTION_SELECTOR = 'button:has-text("weitere"), a:has-text("weitere")'
|
|
||||||
|
|
||||||
|
|
||||||
def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, int, bool]:
|
|
||||||
status, html = 0, ""
|
|
||||||
try:
|
|
||||||
status, html = _fetch_http(url)
|
|
||||||
if not _blocked(status, html):
|
|
||||||
return html, "live_http", status, False
|
|
||||||
except Exception:
|
|
||||||
status, html = 0, ""
|
|
||||||
if playwright:
|
|
||||||
try:
|
|
||||||
status, html = _fetch_playwright(url)
|
|
||||||
if not _blocked(status, html):
|
|
||||||
return html, "live_playwright", status, False
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if html and not is_challenge_html(html) and status and status < 400:
|
|
||||||
return html, "live_http", status, False
|
|
||||||
return html, "blocked", status or 403, True
|
|
||||||
|
|
||||||
|
|
||||||
def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]:
|
|
||||||
status, html = 0, ""
|
|
||||||
if playwright:
|
|
||||||
try:
|
|
||||||
status, html = _fetch_playwright(url, expand_filters=True)
|
|
||||||
if not _blocked(status, html):
|
|
||||||
return html, "live_playwright", status
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
status, html = _fetch_http(url)
|
|
||||||
if not _blocked(status, html):
|
|
||||||
return html, "live_http", status
|
|
||||||
except Exception:
|
|
||||||
status, html = status or 0, html or ""
|
|
||||||
recorded = FIXTURE_DIR / f"{category_ref}-category.html"
|
|
||||||
if recorded.exists():
|
|
||||||
return recorded.read_text(encoding="utf-8"), "recorded_html", 0
|
|
||||||
raise AppError(
|
|
||||||
423,
|
|
||||||
"SOURCE_BLOCKED",
|
|
||||||
"Geizhals blockiert den Taxonomie-Abruf (403/Challenge)",
|
|
||||||
{"url": url, "http_status": status},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_http(url: str) -> tuple[int, str]:
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
response = httpx.get(url, headers=BROWSER_HEADERS, follow_redirects=True, timeout=25.0)
|
|
||||||
return response.status_code, response.text
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, str]:
|
|
||||||
try:
|
|
||||||
from playwright.sync_api import sync_playwright
|
|
||||||
except ImportError as exc:
|
|
||||||
raise AppError(
|
|
||||||
423,
|
|
||||||
"SOURCE_BLOCKED",
|
|
||||||
"Geizhals verlangt einen Browser-Abruf, Playwright ist nicht installiert",
|
|
||||||
) from exc
|
|
||||||
with sync_playwright() as playwright:
|
|
||||||
browser = playwright.chromium.launch(headless=True)
|
|
||||||
page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS)
|
|
||||||
response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
|
||||||
page.wait_for_timeout(4000)
|
|
||||||
if expand_filters:
|
|
||||||
_expand_filter_sections(page)
|
|
||||||
for label in EXPAND_LABELS:
|
|
||||||
button = page.get_by_text(label, exact=False)
|
|
||||||
if button.count():
|
|
||||||
try:
|
|
||||||
button.first.click(timeout=4000)
|
|
||||||
page.wait_for_timeout(800)
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
if expand_filters:
|
|
||||||
_expand_filter_sections(page)
|
|
||||||
_expand_more_options(page)
|
|
||||||
html = page.content()
|
|
||||||
status = response.status if response is not None else 200
|
|
||||||
browser.close()
|
|
||||||
return status, html
|
|
||||||
|
|
||||||
|
|
||||||
def _expand_filter_sections(page: object) -> None:
|
|
||||||
locator = page.locator('button[aria-label*="aufklappen"]')
|
|
||||||
for _ in range(40):
|
|
||||||
if locator.count() == 0:
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
locator.first.click(timeout=3000)
|
|
||||||
page.wait_for_timeout(700)
|
|
||||||
except Exception:
|
|
||||||
return
|
|
||||||
|
|
||||||
|
|
||||||
def _expand_more_options(page: object) -> None:
|
|
||||||
locator = page.locator(MORE_OPTION_SELECTOR)
|
|
||||||
for _ in range(80):
|
|
||||||
if locator.count() == 0:
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
locator.first.click(timeout=2000)
|
|
||||||
page.wait_for_timeout(350)
|
|
||||||
except Exception:
|
|
||||||
return
|
|
||||||
|
|
||||||
|
|
||||||
def _blocked(status: int, html: str) -> bool:
|
|
||||||
if status in {401, 403, 429}:
|
|
||||||
return True
|
|
||||||
return is_challenge_html(html)
|
|
||||||
|
|
@ -1,287 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit
|
|
||||||
|
|
||||||
from app.sources.geizhals.taxonomy_parser import is_challenge_html, node_text, parse_html_tree
|
|
||||||
|
|
||||||
PRODUCT_ID_RE = re.compile(r"-a(\d+)\.html", re.IGNORECASE)
|
|
||||||
PRICE_RE = re.compile(r"(\d{1,3}(?:\.\d{3})+|\d+),(\d{2})")
|
|
||||||
OFFER_RE = re.compile(r"(\d+)\s*Angebote?", re.IGNORECASE)
|
|
||||||
PRODUCT_COUNT_RE = re.compile(r"(\d[\d.]*)\s*Produkte", re.IGNORECASE)
|
|
||||||
CAPACITY_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*(TB|GB)", re.IGNORECASE)
|
|
||||||
WEIGHT_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*kg", re.IGNORECASE)
|
|
||||||
AVAILABILITY_BY_CODE = {"0": "out_of_stock", "1": "limited", "2": "in_stock"}
|
|
||||||
AVAILABILITY_BY_LABEL = {
|
|
||||||
"lagernd": "in_stock",
|
|
||||||
"kurzfristig lieferbar": "limited",
|
|
||||||
"kurzfristig": "limited",
|
|
||||||
"nicht verfügbar": "out_of_stock",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_result_html(html: str, *, page_url: str) -> dict[str, Any]:
|
|
||||||
if is_challenge_html(html):
|
|
||||||
return {"items": [], "page_count": 1, "result_count": 0, "blocked": True}
|
|
||||||
root = parse_html_tree(html)
|
|
||||||
cards = _find_cards(root)
|
|
||||||
items: list[dict[str, Any]] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
for card in cards:
|
|
||||||
item = _parse_card(card, page_url)
|
|
||||||
if item is None or item["source_external_id"] in seen:
|
|
||||||
continue
|
|
||||||
seen.add(item["source_external_id"])
|
|
||||||
items.append(item)
|
|
||||||
page_count = _page_count(root)
|
|
||||||
result_count = _result_count(root) or len(items)
|
|
||||||
return {
|
|
||||||
"items": items,
|
|
||||||
"page_count": page_count,
|
|
||||||
"result_count": result_count,
|
|
||||||
"blocked": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def with_page(url: str, page: int) -> str:
|
|
||||||
parts = urlsplit(url)
|
|
||||||
query = [(name, value) for name, value in parse_qsl(parts.query, keep_blank_values=True) if name != "pg"]
|
|
||||||
if page > 1:
|
|
||||||
query.append(("pg", str(page)))
|
|
||||||
return parts._replace(query=urlencode(query, doseq=True)).geturl()
|
|
||||||
|
|
||||||
|
|
||||||
def _find_cards(root: Any) -> list[Any]:
|
|
||||||
cards: list[Any] = []
|
|
||||||
|
|
||||||
def walk(node: Any) -> None:
|
|
||||||
classes = node.attrs.get("class", "")
|
|
||||||
if "galleryview__item" in classes or "listview__item" in classes:
|
|
||||||
cards.append(node)
|
|
||||||
return
|
|
||||||
for child in node.children:
|
|
||||||
if not isinstance(child, str):
|
|
||||||
walk(child)
|
|
||||||
|
|
||||||
walk(root)
|
|
||||||
return cards
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_card(card: Any, page_url: str) -> dict[str, Any] | None:
|
|
||||||
link = _first_link(card, "name-link") or _first_product_link(card)
|
|
||||||
if link is None:
|
|
||||||
return None
|
|
||||||
href = link.attrs.get("href") or ""
|
|
||||||
match = PRODUCT_ID_RE.search(href)
|
|
||||||
if match is None:
|
|
||||||
return None
|
|
||||||
title = (link.attrs.get("title") or node_text(link)).strip()
|
|
||||||
if not title:
|
|
||||||
return None
|
|
||||||
source_url = urljoin(page_url, href.split("#")[0])
|
|
||||||
raw_attributes = _definition_list(card)
|
|
||||||
sku = _first_text_by_class(card, "galleryview__mpn") or _first_text_by_class(card, "listview__mpn")
|
|
||||||
price_text = _first_text_by_class(card, "price")
|
|
||||||
offers_text = _first_text_by_class(card, "offercount") or ""
|
|
||||||
availability_node = _first_by_attr(card, "data-availability")
|
|
||||||
manufacturer, model = _split_title(title)
|
|
||||||
return {
|
|
||||||
"source_external_id": match.group(1),
|
|
||||||
"source_url": source_url,
|
|
||||||
"source_title": re.sub(r"\s+", " ", title),
|
|
||||||
"manufacturer": manufacturer,
|
|
||||||
"model": model,
|
|
||||||
"manufacturer_sku": sku,
|
|
||||||
"gtin_ean": None,
|
|
||||||
"best_price": _parse_price(price_text),
|
|
||||||
"currency": "EUR" if price_text and "€" in price_text else None,
|
|
||||||
"availability": _availability(availability_node),
|
|
||||||
"seller_count": _seller_count(offers_text),
|
|
||||||
"raw_attributes": raw_attributes,
|
|
||||||
"normalized_attributes": _normalize_attributes(raw_attributes),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _first_link(node: Any, fragment: str) -> Any | None:
|
|
||||||
classes = node.attrs.get("class", "")
|
|
||||||
if node.tag == "a" and fragment in classes:
|
|
||||||
return node
|
|
||||||
for child in node.children:
|
|
||||||
if not isinstance(child, str):
|
|
||||||
found = _first_link(child, fragment)
|
|
||||||
if found is not None:
|
|
||||||
return found
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _first_product_link(node: Any) -> Any | None:
|
|
||||||
if node.tag == "a" and PRODUCT_ID_RE.search(node.attrs.get("href") or ""):
|
|
||||||
href = node.attrs.get("href") or ""
|
|
||||||
if "#ratings" not in href and "#offerlist" not in href:
|
|
||||||
return node
|
|
||||||
for child in node.children:
|
|
||||||
if not isinstance(child, str):
|
|
||||||
found = _first_product_link(child)
|
|
||||||
if found is not None:
|
|
||||||
return found
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _first_text_by_class(node: Any, fragment: str) -> str | None:
|
|
||||||
found = _first_by_class(node, fragment)
|
|
||||||
if found is None:
|
|
||||||
return None
|
|
||||||
text = node_text(found).strip()
|
|
||||||
return re.sub(r"\s+", " ", text) or None
|
|
||||||
|
|
||||||
|
|
||||||
def _first_by_class(node: Any, fragment: str) -> Any | None:
|
|
||||||
if fragment in node.attrs.get("class", ""):
|
|
||||||
return node
|
|
||||||
for child in node.children:
|
|
||||||
if not isinstance(child, str):
|
|
||||||
found = _first_by_class(child, fragment)
|
|
||||||
if found is not None:
|
|
||||||
return found
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _first_by_attr(node: Any, name: str) -> Any | None:
|
|
||||||
if name in node.attrs:
|
|
||||||
return node
|
|
||||||
for child in node.children:
|
|
||||||
if not isinstance(child, str):
|
|
||||||
found = _first_by_attr(child, name)
|
|
||||||
if found is not None:
|
|
||||||
return found
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _definition_list(node: Any) -> dict[str, str]:
|
|
||||||
pairs: dict[str, str] = {}
|
|
||||||
|
|
||||||
def walk(current: Any) -> None:
|
|
||||||
if current.tag == "dl":
|
|
||||||
pending: str | None = None
|
|
||||||
for child in current.children:
|
|
||||||
if isinstance(child, str):
|
|
||||||
continue
|
|
||||||
if child.tag == "dt":
|
|
||||||
pending = re.sub(r"\s+", " ", node_text(child)).strip().rstrip(":")
|
|
||||||
elif child.tag == "dd" and pending:
|
|
||||||
value = re.sub(r"\s+", " ", node_text(child)).strip()
|
|
||||||
if value:
|
|
||||||
pairs[pending] = value
|
|
||||||
pending = None
|
|
||||||
return
|
|
||||||
for child in current.children:
|
|
||||||
if not isinstance(child, str):
|
|
||||||
walk(child)
|
|
||||||
|
|
||||||
walk(node)
|
|
||||||
return pairs
|
|
||||||
|
|
||||||
|
|
||||||
def _split_title(title: str) -> tuple[str | None, str | None]:
|
|
||||||
parts = title.split(None, 1)
|
|
||||||
if not parts:
|
|
||||||
return None, None
|
|
||||||
if len(parts) == 1:
|
|
||||||
return parts[0], None
|
|
||||||
return parts[0], parts[1]
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_price(text: str | None) -> str | None:
|
|
||||||
if not text:
|
|
||||||
return None
|
|
||||||
match = PRICE_RE.search(text.replace("\xa0", " "))
|
|
||||||
if match is None:
|
|
||||||
return None
|
|
||||||
whole = match.group(1).replace(".", "")
|
|
||||||
return f"{whole}.{match.group(2)}"
|
|
||||||
|
|
||||||
|
|
||||||
def _seller_count(text: str) -> int | None:
|
|
||||||
match = OFFER_RE.search(text)
|
|
||||||
if match is None:
|
|
||||||
return None
|
|
||||||
return int(match.group(1))
|
|
||||||
|
|
||||||
|
|
||||||
def _availability(node: Any | None) -> str:
|
|
||||||
if node is None:
|
|
||||||
return "unknown"
|
|
||||||
code = AVAILABILITY_BY_CODE.get(node.attrs.get("data-availability", ""))
|
|
||||||
if code:
|
|
||||||
return code
|
|
||||||
label = (node.attrs.get("aria-label") or node_text(node)).strip().casefold()
|
|
||||||
return AVAILABILITY_BY_LABEL.get(label, "unknown")
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_attributes(raw: dict[str, str]) -> dict[str, Any]:
|
|
||||||
normalized: dict[str, Any] = {}
|
|
||||||
for label, value in raw.items():
|
|
||||||
key = label.casefold()
|
|
||||||
capacity = _capacity_gb(value)
|
|
||||||
if capacity is not None and key in {"ram", "arbeitsspeicher"}:
|
|
||||||
normalized["ram_gb"] = capacity
|
|
||||||
elif capacity is not None and key in {"speicher", "ssd", "festplatte"}:
|
|
||||||
normalized["storage_gb"] = capacity
|
|
||||||
weight = _weight_kg(value)
|
|
||||||
if weight is not None and "gewicht" in key:
|
|
||||||
normalized["weight_kg"] = weight
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
def _capacity_gb(text: str) -> float | int | None:
|
|
||||||
match = CAPACITY_RE.search(text)
|
|
||||||
if match is None:
|
|
||||||
return None
|
|
||||||
value = float(match.group(1).replace(",", "."))
|
|
||||||
if match.group(2).upper() == "TB":
|
|
||||||
value *= 1024
|
|
||||||
return int(value) if value.is_integer() else value
|
|
||||||
|
|
||||||
|
|
||||||
def _weight_kg(text: str) -> float | None:
|
|
||||||
match = WEIGHT_RE.search(text)
|
|
||||||
if match is None:
|
|
||||||
return None
|
|
||||||
return float(match.group(1).replace(",", "."))
|
|
||||||
|
|
||||||
|
|
||||||
def _page_count(root: Any) -> int:
|
|
||||||
pages = {1}
|
|
||||||
|
|
||||||
def walk(node: Any) -> None:
|
|
||||||
href = node.attrs.get("href") or ""
|
|
||||||
if "pg=" in href and ("pagination" in node.attrs.get("class", "") or _in_pagination(node)):
|
|
||||||
for name, value in parse_qsl(urlsplit(href).query):
|
|
||||||
if name == "pg" and value.isdigit():
|
|
||||||
pages.add(int(value))
|
|
||||||
for child in node.children:
|
|
||||||
if not isinstance(child, str):
|
|
||||||
walk(child)
|
|
||||||
|
|
||||||
walk(root)
|
|
||||||
return max(pages)
|
|
||||||
|
|
||||||
|
|
||||||
def _in_pagination(node: Any) -> bool:
|
|
||||||
current = node
|
|
||||||
while current is not None:
|
|
||||||
classes = current.attrs.get("class", "")
|
|
||||||
ident = current.attrs.get("id", "")
|
|
||||||
if "pagination" in classes or ident == "pagination":
|
|
||||||
return True
|
|
||||||
current = current.parent
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _result_count(root: Any) -> int | None:
|
|
||||||
counts = [
|
|
||||||
int(match.replace(".", ""))
|
|
||||||
for match in PRODUCT_COUNT_RE.findall(node_text(root))
|
|
||||||
]
|
|
||||||
return max(counts) if counts else None
|
|
||||||
|
|
@ -1,146 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import parse_qs, urlsplit
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.discovery.enums import FilterType
|
|
||||||
from app.sources.geizhals.live_page import BROWSER_HEADERS
|
|
||||||
from app.sources.geizhals.taxonomy_parser import _group_sort, _option_from_link
|
|
||||||
|
|
||||||
TYPE_MAP = {
|
|
||||||
"enum": FilterType.ENUM.value,
|
|
||||||
"substring": FilterType.ENUM.value,
|
|
||||||
"range_from": FilterType.NUMERIC_THRESHOLD.value,
|
|
||||||
"range_to": FilterType.NUMERIC_THRESHOLD.value,
|
|
||||||
}
|
|
||||||
GROUP_RULES = (
|
|
||||||
("Anschlüsse", ("anschluss", "usb", "thunderbolt", "docking")),
|
|
||||||
("Display", ("display", "pixeldichte", "screen-to-body", "bildwiederhol", "farbraum", "farbräume")),
|
|
||||||
("CPU", ("cpu", "ai-rechen")),
|
|
||||||
("RAM erweitert", ("ram",)),
|
|
||||||
("Speicher", ("speicher", "ssd", "hdd", "flash", "m.2", "cardreader", "laufwerk")),
|
|
||||||
("Grafik", ("grafik", "gpu", "igpu", "directx")),
|
|
||||||
("Wireless", ("wlan", "wifi", "bluetooth", "wireless")),
|
|
||||||
("Mobilfunk", ("mobilfunk", "navigation")),
|
|
||||||
("Akku", ("akku", "netzteil")),
|
|
||||||
("Abmessungen", ("breite", "tiefe", "höhe", "volumen", "gewicht", "verhältnis")),
|
|
||||||
("Eingabe", ("eingabe", "tastatur", "touchpad", "webcam", "kamera")),
|
|
||||||
("Betriebssystem", ("betriebssystem", "sprachversion", "chrome auto")),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def categorylist_api_url(source_url: str) -> str:
|
|
||||||
parts = urlsplit(source_url)
|
|
||||||
query = parse_qs(parts.query)
|
|
||||||
category = (query.get("cat") or [""])[0] or "nb"
|
|
||||||
hloc = (query.get("hloc") or ["de"])[0]
|
|
||||||
return (
|
|
||||||
"https://geizhals.de/api/gh0/categorylist/"
|
|
||||||
f"?cat={category}&xf=&sort=t&hloc={hloc}&v=e&filters_only=1"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_taxonomy_api(source_url: str, *, category_ref: str) -> dict[str, Any] | None:
|
|
||||||
url = categorylist_api_url(source_url)
|
|
||||||
try:
|
|
||||||
response = httpx.get(
|
|
||||||
url,
|
|
||||||
headers={**BROWSER_HEADERS, "Accept": "application/json"},
|
|
||||||
follow_redirects=True,
|
|
||||||
timeout=25.0,
|
|
||||||
)
|
|
||||||
if response.status_code >= 400:
|
|
||||||
return None
|
|
||||||
return parse_taxonomy_api(
|
|
||||||
response.json(),
|
|
||||||
category_ref=category_ref,
|
|
||||||
source_url=source_url,
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_taxonomy_api(payload: dict[str, Any], *, category_ref: str, source_url: str) -> dict[str, Any]:
|
|
||||||
raw_filters = ((payload.get("response") or {}).get("filters") or [])
|
|
||||||
filters = [_filter_from_api(item) for item in raw_filters if isinstance(item, dict)]
|
|
||||||
filters = [item for item in filters if item["options"] or item.get("range")]
|
|
||||||
if not any(item["external_filter_key"] == "price" for item in filters):
|
|
||||||
filters.append(_price_filter())
|
|
||||||
filters.sort(key=lambda item: (item["group_sort"], item["group_label"].lower(), item["label"].lower()))
|
|
||||||
return {
|
|
||||||
"source": "geizhals",
|
|
||||||
"external_category_key": category_ref,
|
|
||||||
"name": "Notebooks" if category_ref == "nb" else category_ref,
|
|
||||||
"canonical_category": "notebook" if category_ref == "nb" else None,
|
|
||||||
"source_url": source_url,
|
|
||||||
"filters": filters,
|
|
||||||
"blocked": False,
|
|
||||||
"discovery_mode": "live_api",
|
|
||||||
"taxonomy_status": "READY",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _filter_from_api(raw: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
key = str(raw.get("id") or "")
|
|
||||||
label = str(raw.get("title") or key)
|
|
||||||
source_type = str(raw.get("type") or "enum")
|
|
||||||
filter_type = TYPE_MAP.get(source_type, FilterType.ENUM.value)
|
|
||||||
options: list[dict[str, Any]] = []
|
|
||||||
encoding_options: dict[str, str] = {}
|
|
||||||
for value in raw.get("values") or []:
|
|
||||||
query = str(value.get("id") or "")
|
|
||||||
if "_" in query:
|
|
||||||
query = query.split("_", 1)[1].replace("+", " ")
|
|
||||||
option = _option_from_link(str(value.get("name") or query), query)
|
|
||||||
if source_type == "range_from":
|
|
||||||
option["operator"] = ">="
|
|
||||||
elif source_type == "range_to":
|
|
||||||
option["operator"] = "<="
|
|
||||||
options.append(option)
|
|
||||||
encoding_options[option["source_value"]] = query
|
|
||||||
if option.get("normalized_value") is not None:
|
|
||||||
encoding_options[str(option["normalized_value"])] = query
|
|
||||||
group_label = _group_label(label)
|
|
||||||
return {
|
|
||||||
"external_filter_key": key,
|
|
||||||
"label": label,
|
|
||||||
"filter_type": filter_type,
|
|
||||||
"canonical_unit": next((item.get("unit") for item in options if item.get("unit")), None),
|
|
||||||
"group_label": group_label,
|
|
||||||
"group_sort": _group_sort(group_label),
|
|
||||||
"options": options,
|
|
||||||
"encoding": {
|
|
||||||
"url_param": "xf",
|
|
||||||
"encoding_type": "OPTION_TOKEN",
|
|
||||||
"options": encoding_options,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _price_filter() -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"external_filter_key": "price",
|
|
||||||
"label": "Preis",
|
|
||||||
"filter_type": FilterType.NUMERIC_RANGE.value,
|
|
||||||
"canonical_unit": "EUR",
|
|
||||||
"group_label": "Allgemein",
|
|
||||||
"group_sort": _group_sort("Allgemein"),
|
|
||||||
"options": [],
|
|
||||||
"range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"},
|
|
||||||
"encoding": {
|
|
||||||
"url_param": "bpmin",
|
|
||||||
"min_parameter": "bpmin",
|
|
||||||
"max_parameter": "bpmax",
|
|
||||||
"encoding_type": "RANGE_PARAMETERS",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _group_label(title: str) -> str:
|
|
||||||
text = title.casefold()
|
|
||||||
for group, needles in GROUP_RULES:
|
|
||||||
if any(needle in text for needle in needles):
|
|
||||||
return group
|
|
||||||
return "Allgemein"
|
|
||||||
|
|
@ -1,581 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
from html.parser import HTMLParser
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import parse_qsl, unquote, urlparse
|
|
||||||
|
|
||||||
from app.discovery.enums import FilterType
|
|
||||||
|
|
||||||
SKIP_OPTION_LABELS = {
|
|
||||||
"alle",
|
|
||||||
"entfernen",
|
|
||||||
"überspringen",
|
|
||||||
"aktualisieren",
|
|
||||||
"abonnieren",
|
|
||||||
"filter zurücksetzen",
|
|
||||||
"nur beliebteste filter anzeigen",
|
|
||||||
"immer alle filter anzeigen",
|
|
||||||
"filter überspringen",
|
|
||||||
"einfache filter",
|
|
||||||
"erweiterte filter",
|
|
||||||
"weitere",
|
|
||||||
"weniger",
|
|
||||||
"alle anzeigen",
|
|
||||||
"weniger anzeigen",
|
|
||||||
}
|
|
||||||
PRODUCT_COUNT_RE = re.compile(r"\(\d[\d.]*\s*(produkte)?\)\s*$", re.IGNORECASE)
|
|
||||||
OPTION_TAGS = {"a", "button", "label", "input", "span"}
|
|
||||||
GROUP_ORDER = (
|
|
||||||
"Allgemein",
|
|
||||||
"Display",
|
|
||||||
"CPU",
|
|
||||||
"RAM",
|
|
||||||
"RAM erweitert",
|
|
||||||
"Speicher",
|
|
||||||
"Grafik",
|
|
||||||
"Anschlüsse",
|
|
||||||
"Wireless",
|
|
||||||
"Mobilfunk",
|
|
||||||
"Eingabe",
|
|
||||||
"Betriebssystem",
|
|
||||||
"Akku",
|
|
||||||
"Abmessungen",
|
|
||||||
"Sonstiges",
|
|
||||||
)
|
|
||||||
RANGE_MIN_NAMES = {"bpmin", "minp", "prmin"}
|
|
||||||
RANGE_MAX_NAMES = {"bpmax", "maxp", "prmax"}
|
|
||||||
TEXT_NAMES = {"asuch", "fs"}
|
|
||||||
HEADING_TAGS = {"th", "h2", "h3", "h4", "h5", "legend", "dt", "caption", "label", "strong"}
|
|
||||||
NUMBER_RE = re.compile(r"(-?\d+(?:[.,]\d+)?)")
|
|
||||||
THRESHOLD_RE = re.compile(r"^(ab|bis)\s+", re.IGNORECASE)
|
|
||||||
UNIT_RE = re.compile(r"(GB|TB|GHz|MHz|Hz|cd/m²|ppi|kg|\"|″|mm|%|€|EUR)$", re.IGNORECASE)
|
|
||||||
TYPE_MAP = {
|
|
||||||
"text": FilterType.ENUM.value,
|
|
||||||
"color": FilterType.ENUM.value,
|
|
||||||
"text_range": FilterType.NUMERIC_THRESHOLD.value,
|
|
||||||
"numeric_range": FilterType.NUMERIC_RANGE.value,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class _Node:
|
|
||||||
def __init__(self, tag: str, attrs: dict[str, str]) -> None:
|
|
||||||
self.tag = tag
|
|
||||||
self.attrs = attrs
|
|
||||||
self.children: list[_Node | str] = []
|
|
||||||
self.parent: _Node | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class _TreeBuilder(HTMLParser):
|
|
||||||
VOID = {
|
|
||||||
"area",
|
|
||||||
"base",
|
|
||||||
"br",
|
|
||||||
"col",
|
|
||||||
"embed",
|
|
||||||
"hr",
|
|
||||||
"img",
|
|
||||||
"input",
|
|
||||||
"link",
|
|
||||||
"meta",
|
|
||||||
"param",
|
|
||||||
"source",
|
|
||||||
"track",
|
|
||||||
"wbr",
|
|
||||||
}
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
super().__init__(convert_charrefs=True)
|
|
||||||
self.root = _Node("document", {})
|
|
||||||
self._stack = [self.root]
|
|
||||||
|
|
||||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
||||||
node = _Node(tag, {key: value or "" for key, value in attrs})
|
|
||||||
parent = self._stack[-1]
|
|
||||||
node.parent = parent
|
|
||||||
parent.children.append(node)
|
|
||||||
if tag not in self.VOID:
|
|
||||||
self._stack.append(node)
|
|
||||||
|
|
||||||
def handle_endtag(self, tag: str) -> None:
|
|
||||||
for index in range(len(self._stack) - 1, 0, -1):
|
|
||||||
if self._stack[index].tag == tag:
|
|
||||||
del self._stack[index:]
|
|
||||||
break
|
|
||||||
|
|
||||||
def handle_data(self, data: str) -> None:
|
|
||||||
text = data.strip()
|
|
||||||
if text:
|
|
||||||
self._stack[-1].children.append(text)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_html_tree(html: str) -> _Node:
|
|
||||||
builder = _TreeBuilder()
|
|
||||||
builder.feed(html)
|
|
||||||
return builder.root
|
|
||||||
|
|
||||||
|
|
||||||
def node_text(node: _Node) -> str:
|
|
||||||
return _node_text(node)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_taxonomy_html(html: str, *, category_ref: str, source_url: str) -> dict[str, Any]:
|
|
||||||
if is_challenge_html(html):
|
|
||||||
return {
|
|
||||||
"source": "geizhals",
|
|
||||||
"external_category_key": category_ref,
|
|
||||||
"name": category_ref,
|
|
||||||
"source_url": source_url,
|
|
||||||
"filters": [],
|
|
||||||
"blocked": True,
|
|
||||||
}
|
|
||||||
builder = _TreeBuilder()
|
|
||||||
builder.feed(html)
|
|
||||||
groups: dict[str, dict[str, Any]] = {}
|
|
||||||
_collect_declared_filters(builder.root, groups)
|
|
||||||
_collect_xf_links(builder.root, groups)
|
|
||||||
_collect_named_inputs(builder.root, groups)
|
|
||||||
filters = [
|
|
||||||
group
|
|
||||||
for group in groups.values()
|
|
||||||
if _usable_group(group)
|
|
||||||
]
|
|
||||||
for group in filters:
|
|
||||||
section = group.get("group_label") or "Allgemein"
|
|
||||||
group["group_label"] = section
|
|
||||||
group["group_sort"] = _group_sort(section)
|
|
||||||
filters.sort(key=lambda item: (item["group_sort"], item["group_label"].lower(), item["label"].lower(), item["external_filter_key"]))
|
|
||||||
return {
|
|
||||||
"source": "geizhals",
|
|
||||||
"external_category_key": category_ref,
|
|
||||||
"name": _category_name(builder.root, category_ref),
|
|
||||||
"canonical_category": "notebook" if category_ref == "nb" else None,
|
|
||||||
"source_url": source_url,
|
|
||||||
"filters": filters,
|
|
||||||
"blocked": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def is_challenge_html(html: str) -> bool:
|
|
||||||
sample = html.lower()
|
|
||||||
markers = (
|
|
||||||
"cf-mitigated",
|
|
||||||
"cf-challenge",
|
|
||||||
"just a moment",
|
|
||||||
"enable javascript and cookies",
|
|
||||||
"checking your browser",
|
|
||||||
"verification successful. waiting for geizhals",
|
|
||||||
)
|
|
||||||
return any(marker in sample for marker in markers)
|
|
||||||
|
|
||||||
|
|
||||||
def _collect_declared_filters(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
|
|
||||||
filter_id = node.attrs.get("data-filter-id")
|
|
||||||
if filter_id:
|
|
||||||
source_type = node.attrs.get("data-filter-type") or ""
|
|
||||||
label = _declared_label(node) or f"Filter {filter_id}"
|
|
||||||
group = groups.setdefault(filter_id, _empty_group(filter_id, label, source_type))
|
|
||||||
section = _section_label(node)
|
|
||||||
if section:
|
|
||||||
group["group_label"] = section
|
|
||||||
if label and group["label"].startswith("Filter "):
|
|
||||||
group["label"] = label
|
|
||||||
if source_type:
|
|
||||||
group["filter_type"] = TYPE_MAP.get(source_type, group["filter_type"])
|
|
||||||
if source_type == "numeric_range":
|
|
||||||
group["encoding"]["encoding_type"] = "RANGE_TOKEN"
|
|
||||||
group["encoding"]["value_template"] = "{min}*{max}"
|
|
||||||
group.setdefault("range", {"minimum": None, "maximum": None, "step": None, "unit": group.get("canonical_unit")})
|
|
||||||
_collect_xf_links(node, groups, required_key=filter_id, allow_histogram=source_type == "numeric_range")
|
|
||||||
return
|
|
||||||
for child in node.children:
|
|
||||||
if isinstance(child, _Node):
|
|
||||||
_collect_declared_filters(child, groups)
|
|
||||||
|
|
||||||
|
|
||||||
def _collect_xf_links(
|
|
||||||
node: _Node,
|
|
||||||
groups: dict[str, dict[str, Any]],
|
|
||||||
required_key: str | None = None,
|
|
||||||
allow_histogram: bool = False,
|
|
||||||
) -> None:
|
|
||||||
if node.tag in OPTION_TAGS:
|
|
||||||
label = node.attrs.get("aria-label") or node.attrs.get("title") or _node_text(node)
|
|
||||||
if _usable_option(node, label, allow_histogram=allow_histogram):
|
|
||||||
for key, query_value in _node_xf_tokens(node):
|
|
||||||
if required_key is not None and key != required_key:
|
|
||||||
continue
|
|
||||||
if required_key is None and not _has_structural_label(node):
|
|
||||||
continue
|
|
||||||
filter_label = _declared_label(_filter_root(node)) or _nearest_group_label(node)
|
|
||||||
group = groups.setdefault(key, _empty_group(key, filter_label, None))
|
|
||||||
section = _section_label(node)
|
|
||||||
if section:
|
|
||||||
group["group_label"] = section
|
|
||||||
if filter_label and (group["label"].startswith("Filter ") or group["label"] == key):
|
|
||||||
group["label"] = filter_label
|
|
||||||
option = _option_from_link(label, query_value)
|
|
||||||
existing = {item["source_value"] for item in group["options"]}
|
|
||||||
if option["source_value"] not in existing:
|
|
||||||
group["options"].append(option)
|
|
||||||
group["encoding"].setdefault("options", {})
|
|
||||||
group["encoding"]["options"][option["source_value"]] = query_value
|
|
||||||
if option.get("normalized_value") is not None:
|
|
||||||
group["encoding"]["options"][str(option["normalized_value"])] = query_value
|
|
||||||
if group["filter_type"] not in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}:
|
|
||||||
group["filter_type"] = _classify_options(group["options"])
|
|
||||||
if group["canonical_unit"] is None:
|
|
||||||
group["canonical_unit"] = option.get("unit")
|
|
||||||
for child in node.children:
|
|
||||||
if isinstance(child, _Node):
|
|
||||||
_collect_xf_links(child, groups, required_key=required_key, allow_histogram=allow_histogram)
|
|
||||||
|
|
||||||
|
|
||||||
def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
|
|
||||||
inputs: list[_Node] = []
|
|
||||||
_find_inputs(node, inputs)
|
|
||||||
names = {item.attrs.get("name", ""): item for item in inputs if item.attrs.get("name")}
|
|
||||||
if names.keys() & RANGE_MIN_NAMES or names.keys() & RANGE_MAX_NAMES:
|
|
||||||
min_name = next((name for name in RANGE_MIN_NAMES if name in names), "bpmin")
|
|
||||||
max_name = next((name for name in RANGE_MAX_NAMES if name in names), "bpmax")
|
|
||||||
label = _nearest_group_label(names.get(min_name) or names.get(max_name) or node) or "Preis"
|
|
||||||
section = _section_label(names.get(min_name) or names.get(max_name) or node) or "Allgemein"
|
|
||||||
groups["price"] = {
|
|
||||||
"external_filter_key": "price",
|
|
||||||
"label": "Preis" if "preis" not in label.lower() else label,
|
|
||||||
"filter_type": FilterType.NUMERIC_RANGE.value,
|
|
||||||
"canonical_unit": "EUR",
|
|
||||||
"group_label": section,
|
|
||||||
"group_sort": _group_sort(section),
|
|
||||||
"options": [],
|
|
||||||
"range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"},
|
|
||||||
"encoding": {
|
|
||||||
"url_param": min_name,
|
|
||||||
"min_parameter": min_name,
|
|
||||||
"max_parameter": max_name,
|
|
||||||
"encoding_type": "RANGE_PARAMETERS",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name in TEXT_NAMES:
|
|
||||||
if name not in names:
|
|
||||||
continue
|
|
||||||
label = _nearest_group_label(names[name]) or "Suchbegriffe"
|
|
||||||
section = _section_label(names[name]) or "Allgemein"
|
|
||||||
groups[name] = {
|
|
||||||
"external_filter_key": name,
|
|
||||||
"label": label,
|
|
||||||
"filter_type": FilterType.TEXT.value,
|
|
||||||
"canonical_unit": None,
|
|
||||||
"group_label": section,
|
|
||||||
"group_sort": _group_sort(section),
|
|
||||||
"options": [],
|
|
||||||
"encoding": {"url_param": name, "encoding_type": "TEXT_PARAMETER", "value_template": "{value}"},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _find_inputs(node: _Node, found: list[_Node]) -> None:
|
|
||||||
if node.tag == "input":
|
|
||||||
found.append(node)
|
|
||||||
for child in node.children:
|
|
||||||
if isinstance(child, _Node):
|
|
||||||
_find_inputs(child, found)
|
|
||||||
|
|
||||||
|
|
||||||
def _node_xf_tokens(node: _Node) -> list[tuple[str, str]]:
|
|
||||||
tokens = _xf_tokens(node.attrs.get("href") or "")
|
|
||||||
if tokens:
|
|
||||||
return tokens
|
|
||||||
for attr in ("data-xf", "data-filter-value", "data-value", "value"):
|
|
||||||
raw = node.attrs.get(attr) or ""
|
|
||||||
if attr == "value" and node.attrs.get("name") not in {"xf", ""} and node.tag != "input":
|
|
||||||
continue
|
|
||||||
parsed = _xf_raw_token(raw)
|
|
||||||
if parsed:
|
|
||||||
return [parsed]
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def _xf_raw_token(raw: str) -> tuple[str, str] | None:
|
|
||||||
value = unquote(raw).replace("|", "~").split("~")[0]
|
|
||||||
if "_" not in value:
|
|
||||||
return None
|
|
||||||
key, token = value.split("_", 1)
|
|
||||||
if key.isdigit() and token:
|
|
||||||
return key, token.replace("+", " ")
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _xf_tokens(href: str) -> list[tuple[str, str]]:
|
|
||||||
if not href or href.startswith("#") or href.startswith("javascript:"):
|
|
||||||
return []
|
|
||||||
query = urlparse(href).query
|
|
||||||
tokens: list[tuple[str, str]] = []
|
|
||||||
for name, value in parse_qsl(query, keep_blank_values=True):
|
|
||||||
if name != "xf" or not value:
|
|
||||||
continue
|
|
||||||
for part in unquote(value).replace("|", "~").split("~"):
|
|
||||||
if "_" not in part:
|
|
||||||
continue
|
|
||||||
key, raw = part.split("_", 1)
|
|
||||||
if key.isdigit() and raw:
|
|
||||||
tokens.append((key, raw.replace("+", " ")))
|
|
||||||
return tokens
|
|
||||||
|
|
||||||
|
|
||||||
def _option_from_link(label: str, query_value: str) -> dict[str, Any]:
|
|
||||||
cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip()
|
|
||||||
operator = None
|
|
||||||
if cleaned.lower().startswith("ab "):
|
|
||||||
operator = ">="
|
|
||||||
elif cleaned.lower().startswith("bis "):
|
|
||||||
operator = "<="
|
|
||||||
number = _first_number(cleaned)
|
|
||||||
unit = _unit(cleaned)
|
|
||||||
return {
|
|
||||||
"source_value": cleaned or query_value,
|
|
||||||
"query_value": query_value,
|
|
||||||
"normalized_value": number,
|
|
||||||
"operator": operator,
|
|
||||||
"unit": unit,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _classify_options(options: list[dict[str, Any]]) -> str:
|
|
||||||
if not options:
|
|
||||||
return FilterType.UNKNOWN.value
|
|
||||||
labels = [str(item["source_value"]).lower() for item in options]
|
|
||||||
if all(item in {"ja", "nein", "egal", "true", "false"} for item in labels):
|
|
||||||
return FilterType.BOOLEAN.value
|
|
||||||
if all(THRESHOLD_RE.match(str(item["source_value"])) for item in options):
|
|
||||||
return FilterType.NUMERIC_THRESHOLD.value
|
|
||||||
return FilterType.ENUM.value
|
|
||||||
|
|
||||||
|
|
||||||
def _empty_group(key: str, label: str | None, source_type: str | None) -> dict[str, Any]:
|
|
||||||
encoding_type = "OPTION_TOKEN"
|
|
||||||
if source_type == "numeric_range":
|
|
||||||
encoding_type = "RANGE_TOKEN"
|
|
||||||
return {
|
|
||||||
"external_filter_key": key,
|
|
||||||
"label": label or f"Filter {key}",
|
|
||||||
"filter_type": TYPE_MAP.get(source_type or "", FilterType.ENUM.value),
|
|
||||||
"canonical_unit": None,
|
|
||||||
"group_label": None,
|
|
||||||
"group_sort": _group_sort("Allgemein"),
|
|
||||||
"options": [],
|
|
||||||
"encoding": {
|
|
||||||
"url_param": "xf",
|
|
||||||
"encoding_type": encoding_type,
|
|
||||||
"options": {},
|
|
||||||
**({"value_template": "{min}*{max}"} if encoding_type == "RANGE_TOKEN" else {}),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _declared_label(node: _Node | None) -> str | None:
|
|
||||||
if node is None:
|
|
||||||
return None
|
|
||||||
attr_id = node.attrs.get("id", "")
|
|
||||||
if attr_id.startswith("filter-element__title-label"):
|
|
||||||
return _clean_label(_node_text(node)) or None
|
|
||||||
for child in node.children:
|
|
||||||
if isinstance(child, _Node):
|
|
||||||
found = _declared_label(child)
|
|
||||||
if found:
|
|
||||||
return found
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _filter_root(node: _Node) -> _Node | None:
|
|
||||||
current: _Node | None = node
|
|
||||||
while current is not None:
|
|
||||||
if current.attrs.get("data-filter-id"):
|
|
||||||
return current
|
|
||||||
current = current.parent
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _nearest_group_label(node: _Node | None) -> str | None:
|
|
||||||
current = node
|
|
||||||
while current is not None:
|
|
||||||
declared = _declared_label(current)
|
|
||||||
if declared:
|
|
||||||
return declared
|
|
||||||
if current.tag in HEADING_TAGS:
|
|
||||||
text = _clean_label(_node_text(current))
|
|
||||||
if text:
|
|
||||||
return text
|
|
||||||
previous = _previous_element(current)
|
|
||||||
while previous is not None:
|
|
||||||
if previous.tag in HEADING_TAGS:
|
|
||||||
text = _clean_label(_node_text(previous))
|
|
||||||
if text:
|
|
||||||
return text
|
|
||||||
previous = _previous_element(previous)
|
|
||||||
labeled = current.attrs.get("aria-label") or current.attrs.get("data-filter-name") or current.attrs.get("title")
|
|
||||||
if labeled:
|
|
||||||
text = _clean_label(labeled)
|
|
||||||
if text and len(text.split()) <= 6:
|
|
||||||
return text
|
|
||||||
current = current.parent
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _has_structural_label(node: _Node) -> bool:
|
|
||||||
if _filter_root(node) is not None:
|
|
||||||
return True
|
|
||||||
current: _Node | None = node
|
|
||||||
while current is not None:
|
|
||||||
if current.tag in {"th", "td", "tr", "table", "dt", "dd"}:
|
|
||||||
return _nearest_group_label(node) is not None
|
|
||||||
current = current.parent
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _usable_option(node: _Node, label: str, *, allow_histogram: bool) -> bool:
|
|
||||||
classes = f"{node.attrs.get('class', '')} {node.attrs.get('id', '')}".lower()
|
|
||||||
if "skip-link" in classes or "add-to-columns" in classes:
|
|
||||||
return False
|
|
||||||
if "bucket" in classes and not allow_histogram:
|
|
||||||
return False
|
|
||||||
cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip()
|
|
||||||
if not cleaned or _skip_option(cleaned):
|
|
||||||
return False
|
|
||||||
if "überspringen" in cleaned.lower() or "aufklappen" in cleaned.lower() or "zuklappen" in cleaned.lower():
|
|
||||||
return False
|
|
||||||
if cleaned.lower().startswith("weitere"):
|
|
||||||
return False
|
|
||||||
if not allow_histogram and re.fullmatch(r"\d[\d.]*\s*produkte", cleaned, re.IGNORECASE):
|
|
||||||
return False
|
|
||||||
if len(cleaned.split()) > 12:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _section_label(node: _Node | None) -> str | None:
|
|
||||||
current = node
|
|
||||||
while current is not None:
|
|
||||||
classes = current.attrs.get("class", "").lower()
|
|
||||||
if "filter-row" in classes or current.attrs.get("data-filter-row-id"):
|
|
||||||
text = _strip_section_suffix(
|
|
||||||
current.attrs.get("data-filter-row-name")
|
|
||||||
or current.attrs.get("aria-label")
|
|
||||||
or _row_heading(current)
|
|
||||||
)
|
|
||||||
if text:
|
|
||||||
return text
|
|
||||||
labeled = current.attrs.get("aria-label") or ""
|
|
||||||
if re.search(r"(auf|zu)klappen", labeled, re.IGNORECASE):
|
|
||||||
text = _strip_section_suffix(labeled)
|
|
||||||
if text:
|
|
||||||
return text
|
|
||||||
current = current.parent
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _row_heading(node: _Node) -> str:
|
|
||||||
for child in node.children:
|
|
||||||
if isinstance(child, _Node) and child.tag in {"button", "h2", "h3", "h4", "legend", "strong"}:
|
|
||||||
text = _clean_label(_node_text(child))
|
|
||||||
if text:
|
|
||||||
return text
|
|
||||||
return _clean_label(node.attrs.get("aria-label") or "")
|
|
||||||
|
|
||||||
|
|
||||||
def _strip_section_suffix(text: str | None) -> str:
|
|
||||||
cleaned = _clean_label(text or "")
|
|
||||||
return re.sub(r"\s*(aufklappen|zuklappen|einklappen)$", "", cleaned, flags=re.IGNORECASE).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def _group_sort(label: str) -> int:
|
|
||||||
try:
|
|
||||||
return GROUP_ORDER.index(label)
|
|
||||||
except ValueError:
|
|
||||||
return len(GROUP_ORDER)
|
|
||||||
|
|
||||||
|
|
||||||
def _usable_group(group: dict[str, Any]) -> bool:
|
|
||||||
if group["filter_type"] == FilterType.TEXT.value:
|
|
||||||
return True
|
|
||||||
if group.get("range") and group["filter_type"] == FilterType.NUMERIC_RANGE.value:
|
|
||||||
return True
|
|
||||||
if not group.get("options") and not group.get("range"):
|
|
||||||
return False
|
|
||||||
label = str(group.get("label") or "")
|
|
||||||
if label.startswith("Filter ") and len(group.get("options") or []) < 2:
|
|
||||||
return False
|
|
||||||
if len(label.split()) > 8:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _previous_element(node: _Node) -> _Node | None:
|
|
||||||
if node.parent is None:
|
|
||||||
return None
|
|
||||||
siblings = [child for child in node.parent.children if isinstance(child, _Node)]
|
|
||||||
try:
|
|
||||||
index = siblings.index(node)
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
return siblings[index - 1] if index > 0 else None
|
|
||||||
|
|
||||||
|
|
||||||
def _node_text(node: _Node) -> str:
|
|
||||||
parts: list[str] = []
|
|
||||||
for child in node.children:
|
|
||||||
if isinstance(child, str):
|
|
||||||
parts.append(child)
|
|
||||||
else:
|
|
||||||
parts.append(_node_text(child))
|
|
||||||
return " ".join(part for part in parts if part).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def _clean_label(text: str) -> str:
|
|
||||||
cleaned = re.sub(r"\s+", " ", text).strip().rstrip(":")
|
|
||||||
cleaned = re.sub(r"\s*(überspringen|entfernen)$", "", cleaned, flags=re.IGNORECASE).strip()
|
|
||||||
if not cleaned or _skip_option(cleaned):
|
|
||||||
return ""
|
|
||||||
return cleaned
|
|
||||||
|
|
||||||
|
|
||||||
def _skip_option(label: str) -> bool:
|
|
||||||
return label.strip().lower() in SKIP_OPTION_LABELS
|
|
||||||
|
|
||||||
|
|
||||||
def _first_number(text: str) -> float | int | None:
|
|
||||||
match = NUMBER_RE.search(text.replace(",", "."))
|
|
||||||
if match is None:
|
|
||||||
return None
|
|
||||||
raw = match.group(1)
|
|
||||||
value = float(raw)
|
|
||||||
return int(value) if value.is_integer() else value
|
|
||||||
|
|
||||||
|
|
||||||
def _unit(text: str) -> str | None:
|
|
||||||
match = UNIT_RE.search(text.replace(" ", ""))
|
|
||||||
if match is None:
|
|
||||||
if "kg" in text.lower():
|
|
||||||
return "kg"
|
|
||||||
return None
|
|
||||||
unit = match.group(1)
|
|
||||||
if unit in {'"', "″"}:
|
|
||||||
return "inch"
|
|
||||||
return unit
|
|
||||||
|
|
||||||
|
|
||||||
def _category_name(root: _Node, fallback: str) -> str:
|
|
||||||
titles: list[str] = []
|
|
||||||
|
|
||||||
def walk(node: _Node) -> None:
|
|
||||||
if node.tag in {"h1", "title"}:
|
|
||||||
text = _node_text(node)
|
|
||||||
if text:
|
|
||||||
titles.append(text)
|
|
||||||
for child in node.children:
|
|
||||||
if isinstance(child, _Node):
|
|
||||||
walk(child)
|
|
||||||
|
|
||||||
walk(root)
|
|
||||||
if titles:
|
|
||||||
return titles[0].split("»")[0].strip() or fallback
|
|
||||||
return "Notebooks" if fallback == "nb" else fallback
|
|
||||||
|
|
@ -1,110 +0,0 @@
|
||||||
import json
|
|
||||||
from datetime import datetime
|
|
||||||
from decimal import Decimal
|
|
||||||
from pathlib import Path
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from app.errors import AppError
|
|
||||||
from app.sources.base import (
|
|
||||||
SourceAttribute,
|
|
||||||
SourceOffer,
|
|
||||||
SourcePriceObservation,
|
|
||||||
SourceProduct,
|
|
||||||
SourceVariant,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_dt(value: str) -> datetime:
|
|
||||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
||||||
|
|
||||||
|
|
||||||
class ManualSource:
|
|
||||||
source_key = "manual"
|
|
||||||
|
|
||||||
def __init__(self, path: Path) -> None:
|
|
||||||
self.path = path
|
|
||||||
self._data = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
self._products = {item["external_id"]: item for item in self._data["products"]}
|
|
||||||
self._offers = {item["external_id"]: item for item in self._data.get("offers", [])}
|
|
||||||
self._prices = self._data.get("prices", [])
|
|
||||||
|
|
||||||
def list_external_product_ids(self) -> list[str]:
|
|
||||||
return list(self._products)
|
|
||||||
|
|
||||||
def get_product(self, external_id: str) -> SourceProduct:
|
|
||||||
raw = self._products.get(external_id)
|
|
||||||
if raw is None:
|
|
||||||
raise AppError(404, "NOT_FOUND", "manual product not found", {"external_id": external_id})
|
|
||||||
variants = []
|
|
||||||
for variant in raw["variants"]:
|
|
||||||
attributes = tuple(
|
|
||||||
SourceAttribute(
|
|
||||||
id=UUID(item["id"]),
|
|
||||||
attribute=item["attribute"],
|
|
||||||
raw_value=item["raw_value"],
|
|
||||||
normalized_value=item.get("normalized_value"),
|
|
||||||
unit=item.get("unit"),
|
|
||||||
normalization_status=item["normalization_status"],
|
|
||||||
normalizer_version=item.get("normalizer_version", "manual-v1"),
|
|
||||||
confidence=Decimal(str(item["confidence"])),
|
|
||||||
source_reference=item["source_reference"],
|
|
||||||
observed_at=_parse_dt(item["observed_at"]),
|
|
||||||
normalization_error=item.get("normalization_error"),
|
|
||||||
)
|
|
||||||
for item in variant["attributes"]
|
|
||||||
)
|
|
||||||
variants.append(
|
|
||||||
SourceVariant(
|
|
||||||
id=UUID(variant["id"]),
|
|
||||||
name=variant["name"],
|
|
||||||
manufacturer_sku=variant.get("manufacturer_sku"),
|
|
||||||
identifiers=variant.get("identifiers") or {},
|
|
||||||
attributes=attributes,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return SourceProduct(
|
|
||||||
source_key=self.source_key,
|
|
||||||
external_id=external_id,
|
|
||||||
id=UUID(raw["id"]),
|
|
||||||
category=raw["category"],
|
|
||||||
brand=raw["brand"],
|
|
||||||
name=raw["name"],
|
|
||||||
model=raw.get("model"),
|
|
||||||
variants=tuple(variants),
|
|
||||||
)
|
|
||||||
|
|
||||||
def get_offers(self, external_product_id: str) -> list[SourceOffer]:
|
|
||||||
product = self.get_product(external_product_id)
|
|
||||||
variant_ids = {variant.id for variant in product.variants}
|
|
||||||
return [
|
|
||||||
SourceOffer(
|
|
||||||
id=UUID(item["id"]),
|
|
||||||
variant_id=UUID(item["variant_id"]),
|
|
||||||
external_id=item["external_id"],
|
|
||||||
seller=item["seller"],
|
|
||||||
url=item.get("url"),
|
|
||||||
market=item["market"],
|
|
||||||
currency=item["currency"],
|
|
||||||
condition=item["condition"],
|
|
||||||
)
|
|
||||||
for item in self._offers.values()
|
|
||||||
if UUID(item["variant_id"]) in variant_ids
|
|
||||||
]
|
|
||||||
|
|
||||||
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]:
|
|
||||||
return [
|
|
||||||
SourcePriceObservation(
|
|
||||||
id=UUID(item["id"]) if item.get("id") else None,
|
|
||||||
external_offer_id=external_offer_id,
|
|
||||||
external_observation_id=item["external_observation_id"],
|
|
||||||
observed_at=_parse_dt(item["observed_at"]),
|
|
||||||
item_price=Decimal(str(item["item_price"])),
|
|
||||||
shipping_price=None if item.get("shipping_price") is None else Decimal(str(item["shipping_price"])),
|
|
||||||
currency=item["currency"],
|
|
||||||
availability=item["availability"],
|
|
||||||
raw_payload=item["raw_payload"],
|
|
||||||
confidence=Decimal(str(item["confidence"])),
|
|
||||||
)
|
|
||||||
for item in self._prices
|
|
||||||
if item["external_offer_id"] == external_offer_id
|
|
||||||
]
|
|
||||||
|
|
@ -1,137 +0,0 @@
|
||||||
<!doctype html>
|
|
||||||
<html lang="de">
|
|
||||||
<head>
|
|
||||||
<title>Notebooks Preisvergleich Geizhals Deutschland</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Notebooks</h1>
|
|
||||||
<div class="filter-row" data-filter-row-id="allgemein">
|
|
||||||
<button aria-label="Allgemein aufklappen">Allgemein</button>
|
|
||||||
<div data-filter-type="text_range" data-filter-id="17023">
|
|
||||||
<span id="filter-element__title-label--17023">RAM</span>
|
|
||||||
<a href="?cat=nb&xf=17023_ab+16GB&promode=true">ab 16GB</a>
|
|
||||||
<a href="?cat=nb&xf=17023_ab+32GB&promode=true">ab 32GB</a>
|
|
||||||
</div>
|
|
||||||
<div data-filter-type="text" data-filter-id="1">
|
|
||||||
<span id="filter-element__title-label--1">Hersteller</span>
|
|
||||||
<a href="?cat=nb&xf=1_Apple" aria-label="Apple (234 Produkte)">Apple</a>
|
|
||||||
<a href="?cat=nb&xf=1_ASUS">ASUS</a>
|
|
||||||
<button type="button" data-xf="1_Lenovo">Lenovo</button>
|
|
||||||
<button type="button">weitere 12</button>
|
|
||||||
</div>
|
|
||||||
<form>
|
|
||||||
<label>Suchbegriffe</label>
|
|
||||||
<input name="asuch" type="text">
|
|
||||||
<label>Preisbereich (€)</label>
|
|
||||||
<input name="bpmin" type="number" placeholder="Preis ab">
|
|
||||||
<input name="bpmax" type="number" placeholder="Preis bis">
|
|
||||||
</form>
|
|
||||||
</div>
|
|
||||||
<div class="filter-row" data-filter-row-id="display">
|
|
||||||
<button aria-label="Display aufklappen">Display</button>
|
|
||||||
<table>
|
|
||||||
<tr>
|
|
||||||
<th>Display-Größe:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=12128_13">13.x"</a>
|
|
||||||
<a href="?cat=nb&xf=12128_14">14.x"</a>
|
|
||||||
<a href="?cat=nb&xf=12128_15">15.x"</a>
|
|
||||||
<a href="?cat=nb&xf=12128_16">16.x"</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>Display-Auflösung:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=9_1920x1080">1920x1080 (Full HD)</a>
|
|
||||||
<a href="?cat=nb&xf=9_2560x1600">2560x1600 (WQXGA)</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>CPU-Kerne:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=12822_8">8 (Octa-Core)</a>
|
|
||||||
<a href="?cat=nb&xf=12822_10">10</a>
|
|
||||||
<a href="?cat=nb&xf=12822_14">14-Core</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>CPU-Kerne ab:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=12822_8">ab 8</a>
|
|
||||||
<a href="?cat=nb&xf=12822_10">ab 10</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>CPU-Threads ab:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=12823_12">ab 12</a>
|
|
||||||
<a href="?cat=nb&xf=12823_16">ab 16</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>CPU-Turbotakt:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=9682_4500">ab 4.5GHz</a>
|
|
||||||
<a href="?cat=nb&xf=9682_5000">ab 5GHz</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>RAM ab:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=12_16384">ab 16GB</a>
|
|
||||||
<a href="?cat=nb&xf=12_32768">ab 32GB</a>
|
|
||||||
<a href="?cat=nb&xf=12_65536">ab 64GB</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>RAM-Typ:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=7520_DDR5">DDR5</a>
|
|
||||||
<a href="?cat=nb&xf=7520_LPDDR5">LPDDR5</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>Speichertyp:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=82_SSD">SSD</a>
|
|
||||||
<a href="?cat=nb&xf=82_M.2">M.2</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>SSD-Kapazität:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=2991_1000">ab 1TB</a>
|
|
||||||
<a href="?cat=nb&xf=2991_2000">ab 2TB</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>Grafik-Typ:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=8149_zzx">dedizierte Grafik (dGPU)</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>GPU - Speichergröße ab:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=7436_2048">ab 2GB</a>
|
|
||||||
<a href="?cat=nb&xf=7436_4096">ab 4GB</a>
|
|
||||||
<a href="?cat=nb&xf=7436_8192">ab 8GB</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>GPU - Speichergröße:</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=16692_8192">8GB</a>
|
|
||||||
<a href="?cat=nb&xf=16692_12288">12GB</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>USB-C mit DisplayPort gesamt (inkl. Thunderbolt):</th>
|
|
||||||
<td>
|
|
||||||
<a href="?cat=nb&xf=13731_1">ab 1x</a>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,52 +0,0 @@
|
||||||
<!doctype html>
|
|
||||||
<html lang="de">
|
|
||||||
<body>
|
|
||||||
<a href="#productlist">6431 Produkte</a>
|
|
||||||
<nav class="pagination">
|
|
||||||
<a class="pagination__page" href="?cat=nb&pg=2">Seite 2</a>
|
|
||||||
<a class="pagination__page last-page" href="?cat=nb&pg=4">Seite 4</a>
|
|
||||||
</nav>
|
|
||||||
<div class="galleryview__item card">
|
|
||||||
<h3 class="galleryview__name">
|
|
||||||
<a class="galleryview__name-link" href="acme-notebook-14-a1000001.html" title="Acme Notebook 14, 32GB RAM, 1TB SSD">
|
|
||||||
Acme Notebook 14, 32GB RAM, 1TB SSD
|
|
||||||
</a>
|
|
||||||
</h3>
|
|
||||||
<span class="galleryview__mpn">ACM-14-32</span>
|
|
||||||
<dl>
|
|
||||||
<dt>Betriebssystem</dt><dd>Linux</dd>
|
|
||||||
<dt>Display</dt><dd>14"</dd>
|
|
||||||
<dt>CPU</dt><dd>Acme 8-Core</dd>
|
|
||||||
<dt>RAM</dt><dd>32GB</dd>
|
|
||||||
<dt>Speicher</dt><dd>1TB SSD</dd>
|
|
||||||
<dt>Grafik</dt><dd>Acme GPU</dd>
|
|
||||||
</dl>
|
|
||||||
<div class="galleryview__price">
|
|
||||||
<a class="galleryview__price-link" href="acme-notebook-14-a1000001.html#offerlist">
|
|
||||||
<span class="price">€ 1298,00</span>
|
|
||||||
</a>
|
|
||||||
<span class="availability" data-availability="2" aria-label="lagernd"></span>
|
|
||||||
</div>
|
|
||||||
<a class="galleryview__offercount-link">8 Angebote</a>
|
|
||||||
</div>
|
|
||||||
<div class="galleryview__item card">
|
|
||||||
<h3 class="galleryview__name">
|
|
||||||
<a class="galleryview__name-link" href="acme-notebook-13-a1000002.html" title="Acme Notebook 13, 8GB RAM">
|
|
||||||
Acme Notebook 13, 8GB RAM
|
|
||||||
</a>
|
|
||||||
</h3>
|
|
||||||
<span class="galleryview__mpn">ACM-13-8</span>
|
|
||||||
<dl>
|
|
||||||
<dt>RAM</dt><dd>8GB</dd>
|
|
||||||
<dt>Speicher</dt><dd>512GB SSD</dd>
|
|
||||||
</dl>
|
|
||||||
<div class="galleryview__price">
|
|
||||||
<a class="galleryview__price-link" href="acme-notebook-13-a1000002.html#offerlist">
|
|
||||||
<span class="price">€ 899,00</span>
|
|
||||||
</a>
|
|
||||||
<span class="availability" data-availability="1" aria-label="kurzfristig lieferbar"></span>
|
|
||||||
</div>
|
|
||||||
<a class="galleryview__offercount-link">3 Angebote</a>
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
{
|
|
||||||
"page": 1,
|
|
||||||
"page_count": 2,
|
|
||||||
"result_count": 3,
|
|
||||||
"http_status": 200,
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"source_external_id": "gh-nb-a",
|
|
||||||
"source_url": "https://geizhals.de/example-mobile-13-a.html",
|
|
||||||
"source_title": "Example Mobile 13 32GB",
|
|
||||||
"manufacturer": "Example",
|
|
||||||
"model": "Mobile 13",
|
|
||||||
"manufacturer_sku": "EM13-32-1T",
|
|
||||||
"gtin_ean": "4000000000001",
|
|
||||||
"best_price": "1298.00",
|
|
||||||
"currency": "EUR",
|
|
||||||
"availability": "in_stock",
|
|
||||||
"seller_count": 8,
|
|
||||||
"raw_attributes": {"RAM": "32GB"},
|
|
||||||
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"source_external_id": "gh-nb-b",
|
|
||||||
"source_url": "https://geizhals.de/example-mobile-13-b.html",
|
|
||||||
"source_title": "Example Mobile 13 8GB",
|
|
||||||
"manufacturer": "Example",
|
|
||||||
"model": "Mobile 13",
|
|
||||||
"manufacturer_sku": "EM13-8-1T",
|
|
||||||
"gtin_ean": "4000000000002",
|
|
||||||
"best_price": "1298.00",
|
|
||||||
"currency": "EUR",
|
|
||||||
"availability": "in_stock",
|
|
||||||
"seller_count": 5,
|
|
||||||
"raw_attributes": {"RAM": "8GB"},
|
|
||||||
"normalized_attributes": {"ram_gb": 8, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
{
|
|
||||||
"page": 2,
|
|
||||||
"page_count": 2,
|
|
||||||
"result_count": 3,
|
|
||||||
"http_status": 200,
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"source_external_id": "gh-nb-c",
|
|
||||||
"source_url": "https://geizhals.de/example-mobile-13-c.html",
|
|
||||||
"source_title": "Example Mobile 13 unknown weight",
|
|
||||||
"manufacturer": "Example",
|
|
||||||
"model": "Mobile 13 UW",
|
|
||||||
"manufacturer_sku": "EM13-32-UW",
|
|
||||||
"gtin_ean": "4000000000003",
|
|
||||||
"best_price": "1298.00",
|
|
||||||
"currency": "EUR",
|
|
||||||
"availability": "in_stock",
|
|
||||||
"seller_count": 3,
|
|
||||||
"raw_attributes": {"RAM": "32GB"},
|
|
||||||
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
{
|
|
||||||
"source": "geizhals",
|
|
||||||
"external_category_key": "nb",
|
|
||||||
"name": "Notebooks",
|
|
||||||
"canonical_category": "notebook",
|
|
||||||
"source_url": "https://geizhals.de/?cat=nb",
|
|
||||||
"filters": [
|
|
||||||
{
|
|
||||||
"external_filter_key": "17023",
|
|
||||||
"label": "Arbeitsspeicher",
|
|
||||||
"filter_type": "NUMERIC_THRESHOLD",
|
|
||||||
"canonical_unit": "GB",
|
|
||||||
"options": [
|
|
||||||
{"source_value": "ab 16GB", "normalized_value": 16, "operator": ">=", "unit": "GB"},
|
|
||||||
{"source_value": "ab 24GB", "normalized_value": 24, "operator": ">=", "unit": "GB"},
|
|
||||||
{"source_value": "ab 32GB", "normalized_value": 32, "operator": ">=", "unit": "GB"}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"external_filter_key": "price",
|
|
||||||
"label": "Preis",
|
|
||||||
"filter_type": "NUMERIC_RANGE",
|
|
||||||
"canonical_unit": "EUR",
|
|
||||||
"range": {"minimum": 0, "maximum": null, "step": 1, "unit": "EUR"}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,80 +0,0 @@
|
||||||
{
|
|
||||||
"products": [
|
|
||||||
{
|
|
||||||
"id": "11111111-1111-4111-8111-111111111111",
|
|
||||||
"external_id": "example-mobile-13",
|
|
||||||
"category": "notebook",
|
|
||||||
"brand": "Example",
|
|
||||||
"name": "Example Mobile 13",
|
|
||||||
"model": "EM13",
|
|
||||||
"variants": [
|
|
||||||
{
|
|
||||||
"id": "22222222-2222-4222-8222-222222222222",
|
|
||||||
"name": "32 GB / 1 TB",
|
|
||||||
"manufacturer_sku": "EM13-32-1T",
|
|
||||||
"identifiers": {"sku": "EM13-32-1T"},
|
|
||||||
"attributes": [
|
|
||||||
{"id": "aaaaaaa1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "aaaaaaa1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "aaaaaaa1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/weight", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "aaaaaaa1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "aaaaaaa1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/display", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "aaaaaaa1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "22222222-2222-4222-8222-22222222222b",
|
|
||||||
"name": "8 GB / 1 TB",
|
|
||||||
"manufacturer_sku": "EM13-8-1T",
|
|
||||||
"identifiers": {"sku": "EM13-8-1T"},
|
|
||||||
"attributes": [
|
|
||||||
{"id": "bbbbbbb1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "8GB RAM", "normalized_value": 8, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "bbbbbbb1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "bbbbbbb1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/weight", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "bbbbbbb1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "bbbbbbb1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/display", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "bbbbbbb1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "22222222-2222-4222-8222-22222222222c",
|
|
||||||
"name": "32 GB / unknown weight",
|
|
||||||
"manufacturer_sku": "EM13-32-UW",
|
|
||||||
"identifiers": {},
|
|
||||||
"attributes": [
|
|
||||||
{"id": "ccccccc1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "ccccccc1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "ccccccc1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "weight unknown", "normalized_value": null, "unit": "kg", "normalization_status": "unknown", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/weight", "observed_at": "2026-10-03T09:00:00Z", "normalization_error": "manual unknown"},
|
|
||||||
{"id": "ccccccc1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "ccccccc1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/display", "observed_at": "2026-10-03T09:00:00Z"},
|
|
||||||
{"id": "ccccccc1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"offers": [
|
|
||||||
{"id": "33333333-3333-4333-8333-333333333333", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-a", "seller": "DemoShop", "url": "https://example.test/a", "market": "DE", "currency": "EUR", "condition": "new"},
|
|
||||||
{"id": "33333333-3333-4333-8333-33333333333b", "variant_id": "22222222-2222-4222-8222-22222222222b", "external_id": "demo-offer-b", "seller": "DemoShop", "url": "https://example.test/b", "market": "DE", "currency": "EUR", "condition": "new"},
|
|
||||||
{"id": "33333333-3333-4333-8333-33333333333c", "variant_id": "22222222-2222-4222-8222-22222222222c", "external_id": "demo-offer-c", "seller": "DemoShop", "url": "https://example.test/c", "market": "DE", "currency": "EUR", "condition": "new"},
|
|
||||||
{"id": "33333333-3333-4333-8333-33333333333d", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-d", "seller": "OtherShop", "url": "https://example.test/d", "market": "DE", "currency": "EUR", "condition": "new"}
|
|
||||||
],
|
|
||||||
"prices": [
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000a1", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000a2", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000a3", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000a4", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000b1", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000b2", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000b3", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000b4", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000c1", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000c2", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000c3", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000c4", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000d1", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000d2", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000d3", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
|
||||||
{"id": "ddddddd1-0000-4000-8000-0000000000d4", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "2049.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "2049.00"}}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,233 +0,0 @@
|
||||||
"""Global product/price schema for v0.1. No user or workspace columns.
|
|
||||||
|
|
||||||
Revision ID: 0002_domain_schema
|
|
||||||
Revises: 0001_deploy_skeleton
|
|
||||||
Create Date: 2026-10-03
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
from sqlalchemy.dialects import postgresql
|
|
||||||
|
|
||||||
revision: str = "0002_domain_schema"
|
|
||||||
down_revision: str | None = "0001_deploy_skeleton"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.create_table(
|
|
||||||
"attribute_definitions",
|
|
||||||
sa.Column("key", sa.String(128), primary_key=True),
|
|
||||||
sa.Column("value_type", sa.String(32), nullable=False),
|
|
||||||
sa.Column("canonical_unit", sa.String(32)),
|
|
||||||
sa.Column("minimum", sa.Numeric(12, 4)),
|
|
||||||
sa.Column("maximum", sa.Numeric(12, 4)),
|
|
||||||
sa.Column("description", sa.Text()),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"value_type IN ('number', 'string', 'boolean', 'string_list')",
|
|
||||||
name="ck_attribute_definitions_value_type",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"sources",
|
|
||||||
sa.Column("id", sa.String(64), primary_key=True),
|
|
||||||
sa.Column("name", sa.String(128), nullable=False),
|
|
||||||
sa.Column("adapter_type", sa.String(64), nullable=False),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"scoring_models",
|
|
||||||
sa.Column("version", sa.String(64), primary_key=True),
|
|
||||||
sa.Column("algorithm_id", sa.String(128), nullable=False),
|
|
||||||
sa.Column("definition", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"product_profiles",
|
|
||||||
sa.Column("id", sa.String(128), primary_key=True),
|
|
||||||
sa.Column("version", sa.Integer(), primary_key=True),
|
|
||||||
sa.Column("name", sa.String(256), nullable=False),
|
|
||||||
sa.Column("category", sa.String(128), nullable=False),
|
|
||||||
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
|
|
||||||
sa.Column("definition", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.CheckConstraint("version > 0", name="ck_product_profiles_version"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"products",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("category", sa.String(128), nullable=False),
|
|
||||||
sa.Column("brand", sa.String(128), nullable=False),
|
|
||||||
sa.Column("name", sa.String(256), nullable=False),
|
|
||||||
sa.Column("model", sa.String(128)),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"product_variants",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("product_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("products.id"), nullable=False),
|
|
||||||
sa.Column("name", sa.String(256), nullable=False),
|
|
||||||
sa.Column("manufacturer_sku", sa.String(128)),
|
|
||||||
sa.Column("identifiers", postgresql.JSONB, nullable=False, server_default=sa.text("'{}'::jsonb")),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
)
|
|
||||||
op.create_index("ix_product_variants_product_id", "product_variants", ["product_id"])
|
|
||||||
op.create_table(
|
|
||||||
"variant_attributes",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
|
||||||
sa.Column("attribute_key", sa.String(128), sa.ForeignKey("attribute_definitions.key"), nullable=False),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("raw_value", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("normalized_value", postgresql.JSONB),
|
|
||||||
sa.Column("unit", sa.String(32)),
|
|
||||||
sa.Column("normalization_status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("normalizer_version", sa.String(64), nullable=False),
|
|
||||||
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
|
|
||||||
sa.Column("source_reference", sa.String(256), nullable=False),
|
|
||||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("normalization_error", sa.Text()),
|
|
||||||
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"normalization_status IN ('normalized', 'unknown')",
|
|
||||||
name="ck_variant_attributes_status",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"CREATE INDEX ix_variant_attributes_lookup ON variant_attributes "
|
|
||||||
"(variant_id, attribute_key, observed_at DESC, ingested_at DESC, id DESC)"
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"offers",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("external_id", sa.String(256), nullable=False),
|
|
||||||
sa.Column("seller", sa.String(256), nullable=False),
|
|
||||||
sa.Column("url", sa.Text()),
|
|
||||||
sa.Column("market", sa.String(16), nullable=False),
|
|
||||||
sa.Column("currency", sa.String(3), nullable=False),
|
|
||||||
sa.Column("condition", sa.String(32), nullable=False),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
|
|
||||||
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
|
|
||||||
sa.CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
|
|
||||||
)
|
|
||||||
op.create_index("ix_offers_variant_id", "offers", ["variant_id"])
|
|
||||||
op.create_table(
|
|
||||||
"price_observations",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("external_observation_id", sa.String(256), nullable=False),
|
|
||||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("item_price", sa.Numeric(12, 2), nullable=False),
|
|
||||||
sa.Column("shipping_price", sa.Numeric(12, 2)),
|
|
||||||
sa.Column("currency", sa.String(3), nullable=False),
|
|
||||||
sa.Column("availability", sa.String(32), nullable=False),
|
|
||||||
sa.Column("raw_payload", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
|
|
||||||
sa.UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
|
|
||||||
sa.CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
|
|
||||||
sa.CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
|
|
||||||
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
|
|
||||||
name="ck_price_obs_availability",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"CREATE INDEX ix_price_observations_lookup ON price_observations "
|
|
||||||
"(offer_id, observed_at DESC, ingested_at DESC, id DESC)"
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"evaluations",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
|
||||||
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
|
|
||||||
sa.Column("profile_id", sa.String(128), nullable=False),
|
|
||||||
sa.Column("profile_version", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
|
|
||||||
sa.Column("as_of", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("snapshot_cutoff", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("scoring_hash", sa.String(64), nullable=False),
|
|
||||||
sa.Column("engine_build", sa.String(64), nullable=False),
|
|
||||||
sa.Column("product_eligibility", sa.String(16), nullable=False),
|
|
||||||
sa.Column("offer_eligibility", sa.String(16), nullable=False),
|
|
||||||
sa.Column("overall_eligibility", sa.String(16), nullable=False),
|
|
||||||
sa.Column("product_score", sa.Numeric(6, 2)),
|
|
||||||
sa.Column("price_score", sa.Numeric(6, 2)),
|
|
||||||
sa.Column("deal_score", sa.Numeric(6, 2)),
|
|
||||||
sa.Column("classification", sa.String(32), nullable=False),
|
|
||||||
sa.Column("price_reference_scope", sa.String(32), nullable=False),
|
|
||||||
sa.Column("input_snapshot", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("result_snapshot", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["profile_id", "profile_version"],
|
|
||||||
["product_profiles.id", "product_profiles.version"],
|
|
||||||
name="fk_evaluations_profile",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
|
||||||
name="ck_eval_product_eligibility",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
|
||||||
name="ck_eval_offer_eligibility",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
|
||||||
name="ck_eval_overall_eligibility",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
|
|
||||||
name="ck_eval_product_score",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
|
|
||||||
name="ck_eval_price_score",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint(
|
|
||||||
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
|
|
||||||
name="ck_eval_deal_score",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.create_index("ix_evaluations_variant_id", "evaluations", ["variant_id"])
|
|
||||||
op.execute(
|
|
||||||
"CREATE INDEX ix_evaluations_profile_created ON evaluations "
|
|
||||||
"(profile_id, profile_version, created_at DESC)"
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"evaluation_components",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("evaluation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("evaluations.id"), nullable=False),
|
|
||||||
sa.Column("kind", sa.String(64), nullable=False),
|
|
||||||
sa.Column("component_key", sa.String(128), nullable=False),
|
|
||||||
sa.Column("raw_score", sa.Numeric(12, 6)),
|
|
||||||
sa.Column("weight", sa.Numeric(12, 6)),
|
|
||||||
sa.Column("contribution", sa.Numeric(12, 6)),
|
|
||||||
sa.Column("details", postgresql.JSONB, nullable=False),
|
|
||||||
sa.UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_table("evaluation_components")
|
|
||||||
op.drop_table("evaluations")
|
|
||||||
op.drop_table("price_observations")
|
|
||||||
op.drop_table("offers")
|
|
||||||
op.drop_table("variant_attributes")
|
|
||||||
op.drop_table("product_variants")
|
|
||||||
op.drop_table("products")
|
|
||||||
op.drop_table("product_profiles")
|
|
||||||
op.drop_table("scoring_models")
|
|
||||||
op.drop_table("sources")
|
|
||||||
op.drop_table("attribute_definitions")
|
|
||||||
|
|
@ -1,229 +0,0 @@
|
||||||
"""Discovery, taxonomy, search runs, and project candidates.
|
|
||||||
|
|
||||||
Revision ID: 0003_discovery_schema
|
|
||||||
Revises: 0002_domain_schema
|
|
||||||
Create Date: 2026-10-04
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
from sqlalchemy.dialects import postgresql
|
|
||||||
|
|
||||||
revision: str = "0003_discovery_schema"
|
|
||||||
down_revision: str | None = "0002_domain_schema"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.create_table(
|
|
||||||
"buying_projects",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("name", sa.String(256), nullable=False),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("workspace_id", sa.String(64)),
|
|
||||||
sa.Column("profile_id", sa.String(128), nullable=False),
|
|
||||||
sa.Column("profile_version", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("shortlist_size", sa.Integer(), nullable=False, server_default="5"),
|
|
||||||
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("archived_at", sa.DateTime(timezone=True)),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["profile_id", "profile_version"],
|
|
||||||
["product_profiles.id", "product_profiles.version"],
|
|
||||||
name="fk_buying_projects_profile",
|
|
||||||
),
|
|
||||||
sa.CheckConstraint("status IN ('ACTIVE', 'PURCHASED', 'ARCHIVED')", name="ck_buying_projects_status"),
|
|
||||||
sa.CheckConstraint("shortlist_size > 0", name="ck_buying_projects_shortlist"),
|
|
||||||
sa.CheckConstraint("schedule_interval_minutes > 0", name="ck_buying_projects_interval"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"buying_project_criteria",
|
|
||||||
sa.Column("project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), primary_key=True),
|
|
||||||
sa.Column("version", sa.Integer(), primary_key=True),
|
|
||||||
sa.Column("definition", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.CheckConstraint("version > 0", name="ck_project_criteria_version"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"source_categories",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("external_category_key", sa.String(64), nullable=False),
|
|
||||||
sa.Column("name", sa.String(256), nullable=False),
|
|
||||||
sa.Column("source_url", sa.Text(), nullable=False),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("last_verified_at", sa.DateTime(timezone=True)),
|
|
||||||
sa.UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"source_filter_definitions",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
|
||||||
sa.Column("external_filter_key", sa.String(64), nullable=False),
|
|
||||||
sa.Column("label", sa.String(256), nullable=False),
|
|
||||||
sa.Column("filter_type", sa.String(32), nullable=False),
|
|
||||||
sa.Column("canonical_unit", sa.String(32)),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"source_filter_options",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), nullable=False),
|
|
||||||
sa.Column("source_value", sa.String(256), nullable=False),
|
|
||||||
sa.Column("normalized_value", postgresql.JSONB),
|
|
||||||
sa.Column("operator", sa.String(16)),
|
|
||||||
sa.Column("unit", sa.String(32)),
|
|
||||||
sa.UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"source_filter_ranges",
|
|
||||||
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), primary_key=True),
|
|
||||||
sa.Column("minimum", sa.Numeric(12, 2)),
|
|
||||||
sa.Column("maximum", sa.Numeric(12, 2)),
|
|
||||||
sa.Column("step", sa.Numeric(12, 2)),
|
|
||||||
sa.Column("unit", sa.String(32)),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"criterion_definitions",
|
|
||||||
sa.Column("key", sa.String(128), primary_key=True),
|
|
||||||
sa.Column("value_type", sa.String(32), nullable=False),
|
|
||||||
sa.Column("unit", sa.String(32)),
|
|
||||||
sa.Column("semantic", sa.String(128)),
|
|
||||||
sa.Column("description", sa.Text()),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"criterion_mappings",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
|
||||||
sa.Column("external_filter_key", sa.String(64), nullable=False),
|
|
||||||
sa.Column("source_value", sa.String(256)),
|
|
||||||
sa.Column("encoding", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"discovery_searches",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
|
||||||
sa.Column("criteria_version", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("mapping_version", sa.String(64), nullable=False),
|
|
||||||
sa.Column("generated_url", sa.Text(), nullable=False),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("broad_result_threshold", sa.Integer(), nullable=False, server_default="300"),
|
|
||||||
sa.Column("next_run_at", sa.DateTime(timezone=True)),
|
|
||||||
sa.Column("last_successful_run_at", sa.DateTime(timezone=True)),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"search_runs",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("discovery_search_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("discovery_searches.id"), nullable=False),
|
|
||||||
sa.Column("criteria_version", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("mapping_version", sa.String(64), nullable=False),
|
|
||||||
sa.Column("generated_url", sa.Text(), nullable=False),
|
|
||||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("result_count", sa.Integer(), nullable=False, server_default="0"),
|
|
||||||
sa.Column("page_count", sa.Integer(), nullable=False, server_default="0"),
|
|
||||||
sa.Column("http_status_summary", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("error_code", sa.String(64)),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"source_product_references",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
|
||||||
sa.Column("source_external_id", sa.String(256)),
|
|
||||||
sa.Column("source_url", sa.Text()),
|
|
||||||
sa.Column("source_title", sa.String(512), nullable=False),
|
|
||||||
sa.Column("manufacturer", sa.String(256)),
|
|
||||||
sa.Column("model", sa.String(256)),
|
|
||||||
sa.Column("manufacturer_sku", sa.String(128)),
|
|
||||||
sa.Column("gtin_ean", sa.String(32)),
|
|
||||||
sa.Column("normalized_spec_fingerprint", sa.Text()),
|
|
||||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("lifecycle_status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("match_status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("consecutive_missing_success_runs", sa.Integer(), nullable=False, server_default="0"),
|
|
||||||
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"product_observations",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
|
||||||
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
|
|
||||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("raw_title", sa.String(512), nullable=False),
|
|
||||||
sa.Column("best_price", sa.Numeric(12, 2)),
|
|
||||||
sa.Column("currency", sa.String(3)),
|
|
||||||
sa.Column("raw_attributes", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False),
|
|
||||||
sa.Column("availability", sa.String(32)),
|
|
||||||
sa.Column("seller_count", sa.Integer()),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"search_run_items",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
|
|
||||||
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
|
||||||
sa.Column("observation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_observations.id")),
|
|
||||||
sa.Column("diff_status", sa.String(32), nullable=False),
|
|
||||||
sa.UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"project_candidates",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
|
|
||||||
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
|
||||||
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("match_status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("product_eligibility", sa.String(16)),
|
|
||||||
sa.Column("product_score", sa.Numeric(6, 2)),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"candidate_status_history",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("candidate_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("project_candidates.id"), nullable=False),
|
|
||||||
sa.Column("from_status", sa.String(32)),
|
|
||||||
sa.Column("to_status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("reason", sa.String(128)),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
)
|
|
||||||
op.create_index("ix_discovery_searches_next_run", "discovery_searches", ["next_run_at"])
|
|
||||||
op.create_index("ix_search_runs_search", "search_runs", ["discovery_search_id", "started_at"])
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_table("candidate_status_history")
|
|
||||||
op.drop_table("project_candidates")
|
|
||||||
op.drop_table("search_run_items")
|
|
||||||
op.drop_table("product_observations")
|
|
||||||
op.drop_table("source_product_references")
|
|
||||||
op.drop_table("search_runs")
|
|
||||||
op.drop_table("discovery_searches")
|
|
||||||
op.drop_table("criterion_mappings")
|
|
||||||
op.drop_table("criterion_definitions")
|
|
||||||
op.drop_table("source_filter_ranges")
|
|
||||||
op.drop_table("source_filter_options")
|
|
||||||
op.drop_table("source_filter_definitions")
|
|
||||||
op.drop_table("source_categories")
|
|
||||||
op.drop_table("buying_project_criteria")
|
|
||||||
op.drop_table("buying_projects")
|
|
||||||
|
|
@ -1,127 +0,0 @@
|
||||||
"""Dynamic attribute and criterion value records.
|
|
||||||
|
|
||||||
Revision ID: 0004_dynamic_attributes
|
|
||||||
Revises: 0003_discovery_schema
|
|
||||||
Create Date: 2026-10-04
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
from sqlalchemy.dialects import postgresql
|
|
||||||
|
|
||||||
revision: str = "0004_dynamic_attributes"
|
|
||||||
down_revision: str | None = "0003_discovery_schema"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column("criterion_definitions", sa.Column("category", sa.String(128)))
|
|
||||||
op.add_column("source_categories", sa.Column("canonical_category", sa.String(128)))
|
|
||||||
op.create_table(
|
|
||||||
"project_criteria",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
|
|
||||||
sa.Column("version", sa.Integer(), nullable=False),
|
|
||||||
sa.Column("item_id", sa.String(128), nullable=False),
|
|
||||||
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
|
|
||||||
sa.Column("operator", sa.String(16), nullable=False),
|
|
||||||
sa.Column("value", postgresql.JSONB),
|
|
||||||
sa.Column("mandatory", sa.Boolean(), nullable=False, server_default=sa.true()),
|
|
||||||
sa.ForeignKeyConstraint(
|
|
||||||
["project_id", "version"],
|
|
||||||
["buying_project_criteria.project_id", "buying_project_criteria.version"],
|
|
||||||
name="fk_project_criteria_version",
|
|
||||||
),
|
|
||||||
sa.UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"product_observation_attributes",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column(
|
|
||||||
"observation_id",
|
|
||||||
postgresql.UUID(as_uuid=True),
|
|
||||||
sa.ForeignKey("product_observations.id"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column("attribute_key", sa.String(128), nullable=False),
|
|
||||||
sa.Column("raw_value", postgresql.JSONB),
|
|
||||||
sa.Column("normalized_value", postgresql.JSONB),
|
|
||||||
sa.Column("unit", sa.String(32)),
|
|
||||||
sa.UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),
|
|
||||||
)
|
|
||||||
_migrate_criteria_json()
|
|
||||||
_migrate_observation_json()
|
|
||||||
op.drop_column("buying_project_criteria", "definition")
|
|
||||||
op.drop_column("product_observations", "raw_attributes")
|
|
||||||
op.drop_column("product_observations", "normalized_attributes")
|
|
||||||
|
|
||||||
|
|
||||||
def _migrate_criteria_json() -> None:
|
|
||||||
bind = op.get_bind()
|
|
||||||
rows = bind.execute(sa.text("SELECT project_id, version, definition FROM buying_project_criteria")).mappings()
|
|
||||||
for row in rows:
|
|
||||||
items = (row["definition"] or {}).get("items") or []
|
|
||||||
for raw in items:
|
|
||||||
bind.execute(
|
|
||||||
sa.text(
|
|
||||||
"""
|
|
||||||
INSERT INTO project_criteria
|
|
||||||
(id, project_id, version, item_id, criterion_key, operator, value, mandatory)
|
|
||||||
VALUES
|
|
||||||
(gen_random_uuid(), :project_id, :version, :item_id, :criterion_key, :operator, :value, :mandatory)
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
{
|
|
||||||
"project_id": row["project_id"],
|
|
||||||
"version": row["version"],
|
|
||||||
"item_id": str(raw.get("id") or raw.get("criterion")),
|
|
||||||
"criterion_key": str(raw.get("criterion")),
|
|
||||||
"operator": str(raw.get("operator")),
|
|
||||||
"value": raw.get("value"),
|
|
||||||
"mandatory": bool(raw.get("mandatory", True)),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _migrate_observation_json() -> None:
|
|
||||||
bind = op.get_bind()
|
|
||||||
rows = bind.execute(
|
|
||||||
sa.text("SELECT id, raw_attributes, normalized_attributes FROM product_observations")
|
|
||||||
).mappings()
|
|
||||||
for row in rows:
|
|
||||||
raw = row["raw_attributes"] or {}
|
|
||||||
normalized = row["normalized_attributes"] or {}
|
|
||||||
keys = set(raw) | set(normalized)
|
|
||||||
for key in keys:
|
|
||||||
bind.execute(
|
|
||||||
sa.text(
|
|
||||||
"""
|
|
||||||
INSERT INTO product_observation_attributes
|
|
||||||
(id, observation_id, attribute_key, raw_value, normalized_value, unit)
|
|
||||||
VALUES
|
|
||||||
(gen_random_uuid(), :observation_id, :attribute_key, :raw_value, :normalized_value, NULL)
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
{
|
|
||||||
"observation_id": row["id"],
|
|
||||||
"attribute_key": key,
|
|
||||||
"raw_value": raw.get(key),
|
|
||||||
"normalized_value": normalized.get(key),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.add_column("product_observations", sa.Column("raw_attributes", postgresql.JSONB, nullable=False, server_default="{}"))
|
|
||||||
op.add_column(
|
|
||||||
"product_observations",
|
|
||||||
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False, server_default="{}"),
|
|
||||||
)
|
|
||||||
op.add_column("buying_project_criteria", sa.Column("definition", postgresql.JSONB, nullable=False, server_default='{"items":[]}'))
|
|
||||||
op.drop_table("product_observation_attributes")
|
|
||||||
op.drop_table("project_criteria")
|
|
||||||
op.drop_column("source_categories", "canonical_category")
|
|
||||||
op.drop_column("criterion_definitions", "category")
|
|
||||||
|
|
@ -1,261 +0,0 @@
|
||||||
"""Frontend MVP v0.3 contracts: categories, optional templates, criterion UUIDs.
|
|
||||||
|
|
||||||
Revision ID: 0005_frontend_mvp_v03
|
|
||||||
Revises: 0004_dynamic_attributes
|
|
||||||
Create Date: 2026-10-04
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
from sqlalchemy.dialects import postgresql
|
|
||||||
|
|
||||||
revision: str = "0005_frontend_mvp_v03"
|
|
||||||
down_revision: str | None = "0004_dynamic_attributes"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.create_table(
|
|
||||||
"product_categories",
|
|
||||||
sa.Column("id", sa.String(64), primary_key=True),
|
|
||||||
sa.Column("name", sa.String(256), nullable=False),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False, server_default="ACTIVE"),
|
|
||||||
)
|
|
||||||
op.execute("INSERT INTO product_categories (id, name, status) VALUES ('notebook', 'Notebooks', 'ACTIVE')")
|
|
||||||
|
|
||||||
op.add_column("source_categories", sa.Column("product_category_id", sa.String(64)))
|
|
||||||
op.create_foreign_key(
|
|
||||||
"fk_source_categories_product_category",
|
|
||||||
"source_categories",
|
|
||||||
"product_categories",
|
|
||||||
["product_category_id"],
|
|
||||||
["id"],
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE source_categories
|
|
||||||
SET product_category_id = canonical_category
|
|
||||||
WHERE canonical_category IS NOT NULL
|
|
||||||
AND EXISTS (SELECT 1 FROM product_categories WHERE id = source_categories.canonical_category)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
op.add_column("criterion_definitions", sa.Column("id", postgresql.UUID(as_uuid=True)))
|
|
||||||
op.execute("UPDATE criterion_definitions SET id = gen_random_uuid() WHERE id IS NULL")
|
|
||||||
op.alter_column("criterion_definitions", "id", nullable=False)
|
|
||||||
op.add_column(
|
|
||||||
"criterion_definitions",
|
|
||||||
sa.Column("origin", sa.String(16), nullable=False, server_default="SOURCE"),
|
|
||||||
)
|
|
||||||
op.add_column("criterion_definitions", sa.Column("source", sa.String(64)))
|
|
||||||
op.add_column("criterion_definitions", sa.Column("product_category_id", sa.String(64)))
|
|
||||||
op.add_column("criterion_definitions", sa.Column("label", sa.String(256)))
|
|
||||||
op.add_column("criterion_definitions", sa.Column("scoring_attribute_key", sa.String(128)))
|
|
||||||
op.execute("UPDATE criterion_definitions SET label = key WHERE label IS NULL")
|
|
||||||
op.execute("UPDATE criterion_definitions SET scoring_attribute_key = key")
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE criterion_definitions
|
|
||||||
SET product_category_id = category
|
|
||||||
WHERE category IS NOT NULL
|
|
||||||
AND EXISTS (SELECT 1 FROM product_categories WHERE id = criterion_definitions.category)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
op.alter_column("criterion_definitions", "label", nullable=False)
|
|
||||||
op.create_foreign_key(
|
|
||||||
"fk_criterion_definitions_product_category",
|
|
||||||
"criterion_definitions",
|
|
||||||
"product_categories",
|
|
||||||
["product_category_id"],
|
|
||||||
["id"],
|
|
||||||
)
|
|
||||||
|
|
||||||
op.add_column("project_criteria", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
|
|
||||||
op.add_column(
|
|
||||||
"project_criteria",
|
|
||||||
sa.Column("enabled_for_discovery", sa.Boolean(), nullable=False, server_default=sa.true()),
|
|
||||||
)
|
|
||||||
op.add_column("project_criteria", sa.Column("created_at", sa.DateTime(timezone=True)))
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE project_criteria
|
|
||||||
SET enabled_for_discovery = mandatory,
|
|
||||||
created_at = timezone('utc', now())
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE project_criteria AS pc
|
|
||||||
SET criterion_definition_id = cd.id
|
|
||||||
FROM criterion_definitions AS cd
|
|
||||||
WHERE pc.criterion_key = cd.key
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
op.alter_column("project_criteria", "criterion_definition_id", nullable=False)
|
|
||||||
op.alter_column("project_criteria", "created_at", nullable=False)
|
|
||||||
op.drop_constraint("project_criteria_criterion_key_fkey", "project_criteria", type_="foreignkey")
|
|
||||||
|
|
||||||
op.add_column("criterion_mappings", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE criterion_mappings AS cm
|
|
||||||
SET criterion_definition_id = cd.id
|
|
||||||
FROM criterion_definitions AS cd
|
|
||||||
WHERE cm.criterion_key = cd.key
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
op.alter_column("criterion_mappings", "criterion_definition_id", nullable=False)
|
|
||||||
op.drop_constraint("criterion_mappings_criterion_key_fkey", "criterion_mappings", type_="foreignkey")
|
|
||||||
|
|
||||||
op.drop_constraint("criterion_definitions_pkey", "criterion_definitions", type_="primary")
|
|
||||||
op.create_primary_key("criterion_definitions_pkey", "criterion_definitions", ["id"])
|
|
||||||
op.create_unique_constraint("uq_criterion_definition_key", "criterion_definitions", ["key"])
|
|
||||||
op.create_foreign_key(
|
|
||||||
"fk_project_criteria_definition",
|
|
||||||
"project_criteria",
|
|
||||||
"criterion_definitions",
|
|
||||||
["criterion_definition_id"],
|
|
||||||
["id"],
|
|
||||||
)
|
|
||||||
op.create_foreign_key(
|
|
||||||
"fk_criterion_mappings_definition",
|
|
||||||
"criterion_mappings",
|
|
||||||
"criterion_definitions",
|
|
||||||
["criterion_definition_id"],
|
|
||||||
["id"],
|
|
||||||
)
|
|
||||||
|
|
||||||
op.create_table(
|
|
||||||
"source_attribute_mappings",
|
|
||||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
|
||||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
|
||||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id")),
|
|
||||||
sa.Column("raw_name", sa.String(256), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"criterion_definition_id",
|
|
||||||
postgresql.UUID(as_uuid=True),
|
|
||||||
sa.ForeignKey("criterion_definitions.id"),
|
|
||||||
nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column("status", sa.String(32), nullable=False),
|
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
|
||||||
)
|
|
||||||
|
|
||||||
op.add_column("buying_projects", sa.Column("product_category_id", sa.String(64)))
|
|
||||||
op.add_column("buying_projects", sa.Column("market", sa.String(8)))
|
|
||||||
op.add_column("buying_projects", sa.Column("currency", sa.String(3)))
|
|
||||||
op.add_column("buying_projects", sa.Column("target_budget", sa.Numeric(12, 2)))
|
|
||||||
op.add_column("buying_projects", sa.Column("template_id", sa.String(128)))
|
|
||||||
op.add_column("buying_projects", sa.Column("template_version", sa.Integer()))
|
|
||||||
op.add_column("buying_projects", sa.Column("criteria_version", sa.Integer(), nullable=False, server_default="1"))
|
|
||||||
op.add_column("buying_projects", sa.Column("updated_at", sa.DateTime(timezone=True)))
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE buying_projects
|
|
||||||
SET template_id = profile_id,
|
|
||||||
template_version = profile_version,
|
|
||||||
product_category_id = 'notebook',
|
|
||||||
market = 'DE',
|
|
||||||
currency = 'EUR',
|
|
||||||
updated_at = created_at
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE buying_projects AS bp
|
|
||||||
SET criteria_version = COALESCE((
|
|
||||||
SELECT MAX(version) FROM buying_project_criteria WHERE project_id = bp.id
|
|
||||||
), 1)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
op.alter_column("buying_projects", "product_category_id", nullable=False)
|
|
||||||
op.alter_column("buying_projects", "market", nullable=False)
|
|
||||||
op.alter_column("buying_projects", "currency", nullable=False)
|
|
||||||
op.alter_column("buying_projects", "updated_at", nullable=False)
|
|
||||||
op.create_foreign_key(
|
|
||||||
"fk_buying_projects_product_category",
|
|
||||||
"buying_projects",
|
|
||||||
"product_categories",
|
|
||||||
["product_category_id"],
|
|
||||||
["id"],
|
|
||||||
)
|
|
||||||
op.drop_constraint("fk_buying_projects_profile", "buying_projects", type_="foreignkey")
|
|
||||||
op.drop_constraint("ck_buying_projects_interval", "buying_projects", type_="check")
|
|
||||||
op.drop_column("buying_projects", "profile_id")
|
|
||||||
op.drop_column("buying_projects", "profile_version")
|
|
||||||
op.drop_column("buying_projects", "schedule_interval_minutes")
|
|
||||||
op.create_foreign_key(
|
|
||||||
"fk_buying_projects_template",
|
|
||||||
"buying_projects",
|
|
||||||
"product_profiles",
|
|
||||||
["template_id", "template_version"],
|
|
||||||
["id", "version"],
|
|
||||||
)
|
|
||||||
|
|
||||||
op.add_column(
|
|
||||||
"discovery_searches",
|
|
||||||
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
|
||||||
)
|
|
||||||
op.add_column("search_runs", sa.Column("new_count", sa.Integer(), nullable=False, server_default="0"))
|
|
||||||
op.add_column("search_runs", sa.Column("changed_count", sa.Integer(), nullable=False, server_default="0"))
|
|
||||||
op.add_column("search_runs", sa.Column("unchanged_count", sa.Integer(), nullable=False, server_default="0"))
|
|
||||||
op.add_column("search_runs", sa.Column("missing_count", sa.Integer(), nullable=False, server_default="0"))
|
|
||||||
|
|
||||||
op.add_column(
|
|
||||||
"product_observation_attributes",
|
|
||||||
sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)),
|
|
||||||
)
|
|
||||||
op.add_column("product_observation_attributes", sa.Column("raw_name", sa.String(256)))
|
|
||||||
op.create_foreign_key(
|
|
||||||
"fk_observation_attributes_definition",
|
|
||||||
"product_observation_attributes",
|
|
||||||
"criterion_definitions",
|
|
||||||
["criterion_definition_id"],
|
|
||||||
["id"],
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE product_observation_attributes AS poa
|
|
||||||
SET criterion_definition_id = cd.id,
|
|
||||||
raw_name = poa.attribute_key
|
|
||||||
FROM criterion_definitions AS cd
|
|
||||||
WHERE poa.attribute_key = cd.scoring_attribute_key
|
|
||||||
OR poa.attribute_key = cd.key
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_constraint("fk_observation_attributes_definition", "product_observation_attributes", type_="foreignkey")
|
|
||||||
op.drop_column("product_observation_attributes", "raw_name")
|
|
||||||
op.drop_column("product_observation_attributes", "criterion_definition_id")
|
|
||||||
op.drop_column("search_runs", "missing_count")
|
|
||||||
op.drop_column("search_runs", "unchanged_count")
|
|
||||||
op.drop_column("search_runs", "changed_count")
|
|
||||||
op.drop_column("search_runs", "new_count")
|
|
||||||
op.drop_column("discovery_searches", "schedule_interval_minutes")
|
|
||||||
op.drop_constraint("fk_buying_projects_template", "buying_projects", type_="foreignkey")
|
|
||||||
op.add_column("buying_projects", sa.Column("profile_id", sa.String(128)))
|
|
||||||
op.add_column("buying_projects", sa.Column("profile_version", sa.Integer()))
|
|
||||||
op.add_column(
|
|
||||||
"buying_projects",
|
|
||||||
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
|
||||||
)
|
|
||||||
op.execute("UPDATE buying_projects SET profile_id = template_id, profile_version = template_version")
|
|
||||||
op.alter_column("buying_projects", "profile_id", nullable=False)
|
|
||||||
op.alter_column("buying_projects", "profile_version", nullable=False)
|
|
||||||
op.drop_constraint("fk_buying_projects_product_category", "buying_projects", type_="foreignkey")
|
|
||||||
op.drop_column("buying_projects", "updated_at")
|
|
||||||
op.drop_column("buying_projects", "criteria_version")
|
|
||||||
op.drop_column("buying_projects", "template_version")
|
|
||||||
op.drop_column("buying_projects", "template_id")
|
|
||||||
op.drop_column("buying_projects", "target_budget")
|
|
||||||
op.drop_column("buying_projects", "currency")
|
|
||||||
op.drop_column("buying_projects", "market")
|
|
||||||
op.drop_column("buying_projects", "product_category_id")
|
|
||||||
op.drop_table("source_attribute_mappings")
|
|
||||||
raise NotImplementedError("full downgrade of criterion UUID primary key is not supported")
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
"""Seedable SourceCategory readiness for the notebook pilot.
|
|
||||||
|
|
||||||
Revision ID: 0006_source_category_readiness
|
|
||||||
Revises: 0005_frontend_mvp_v03
|
|
||||||
Create Date: 2026-10-04
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0006_source_category_readiness"
|
|
||||||
down_revision: str | None = "0005_frontend_mvp_v03"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"source_categories",
|
|
||||||
sa.Column("taxonomy_status", sa.String(32), nullable=False, server_default="NOT_LEARNED"),
|
|
||||||
)
|
|
||||||
op.add_column("source_categories", sa.Column("last_taxonomy_scan_at", sa.DateTime(timezone=True)))
|
|
||||||
op.execute(
|
|
||||||
"""
|
|
||||||
UPDATE source_categories
|
|
||||||
SET taxonomy_status = 'READY',
|
|
||||||
last_taxonomy_scan_at = last_verified_at
|
|
||||||
WHERE last_verified_at IS NOT NULL
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("source_categories", "last_taxonomy_scan_at")
|
|
||||||
op.drop_column("source_categories", "taxonomy_status")
|
|
||||||
|
|
@ -1,24 +0,0 @@
|
||||||
"""Persist source filter groups from the learned taxonomy.
|
|
||||||
|
|
||||||
Revision ID: 0007_filter_groups
|
|
||||||
Revises: 0006_source_category_readiness
|
|
||||||
Create Date: 2026-10-04
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections.abc import Sequence
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0007_filter_groups"
|
|
||||||
down_revision: str | None = "0006_source_category_readiness"
|
|
||||||
branch_labels: str | Sequence[str] | None = None
|
|
||||||
depends_on: str | Sequence[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column("source_filter_definitions", sa.Column("group_label", sa.String(128)))
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("source_filter_definitions", "group_label")
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
{
|
|
||||||
"id": "mobile-development-notebook",
|
|
||||||
"version": 1,
|
|
||||||
"name": "Mobiler Entwicklungsrechner",
|
|
||||||
"category": "notebook",
|
|
||||||
"currency": "EUR",
|
|
||||||
"market": "DE",
|
|
||||||
"allowed_conditions": ["new"],
|
|
||||||
"budget": {"target": "1500.00", "maximum": "2000.00"},
|
|
||||||
"scoring_version": "generic-v1",
|
|
||||||
"constraints": [
|
|
||||||
{"id": "min_ram", "attribute": "ram_gb", "operator": ">=", "value": 16, "mandatory": true},
|
|
||||||
{"id": "min_storage", "attribute": "storage_gb", "operator": ">=", "value": 512, "mandatory": true},
|
|
||||||
{"id": "max_weight", "attribute": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": true}
|
|
||||||
],
|
|
||||||
"preferences": [
|
|
||||||
{"id": "ram", "attribute": "ram_gb", "strategy": "numeric_linear", "direction": "higher", "acceptable": 16, "ideal": 32, "weight": 0.30},
|
|
||||||
{"id": "weight", "attribute": "weight_kg", "strategy": "numeric_linear", "direction": "lower", "acceptable": 1.6, "ideal": 1.0, "weight": 0.25},
|
|
||||||
{"id": "battery", "attribute": "battery_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.20},
|
|
||||||
{"id": "display", "attribute": "display_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.15},
|
|
||||||
{"id": "cpu", "attribute": "cpu_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.10}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
{
|
|
||||||
"version": "generic-v1",
|
|
||||||
"algorithm_id": "linear-budget-history-v1",
|
|
||||||
"supported_preference_strategies": ["numeric_linear"],
|
|
||||||
"minimum_confidence": 0.8,
|
|
||||||
"maximum_price_age_hours": 48,
|
|
||||||
"optional_constraint_penalty_points": 5,
|
|
||||||
"missing_preference_score": 0,
|
|
||||||
"product_weight": 0.6,
|
|
||||||
"price_weight": 0.4,
|
|
||||||
"budget_weight": 0.4,
|
|
||||||
"history_weight": 0.6,
|
|
||||||
"history_window_days": 90,
|
|
||||||
"minimum_history_days": 3,
|
|
||||||
"history_discount_for_max_score": 0.2,
|
|
||||||
"classification_upper_bounds": [
|
|
||||||
{"upper": 60, "inclusive": false, "label": "IGNORE"},
|
|
||||||
{"upper": 75, "inclusive": false, "label": "WATCH"},
|
|
||||||
{"upper": 85, "inclusive": false, "label": "GOOD"},
|
|
||||||
{"upper": 92, "inclusive": true, "label": "VERY_GOOD"},
|
|
||||||
{"upper": 100, "inclusive": true, "label": "EXCEPTIONAL"}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.domain.validation import parse_scoring_model
|
|
||||||
from app.persistence.db import SessionLocal
|
|
||||||
from app.persistence.models import ScoringModelRecord, SourceRecord
|
|
||||||
from app.services.catalog_service import ensure_product_category, ensure_source_category
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent.parent
|
|
||||||
RULES = ROOT / "rules" / "generic-v1.json"
|
|
||||||
|
|
||||||
|
|
||||||
def run_bootstrap(session: Session) -> None:
|
|
||||||
_seed_source(session)
|
|
||||||
_seed_scoring_model(session)
|
|
||||||
ensure_product_category(session, "notebook", "Notebooks")
|
|
||||||
ensure_source_category(
|
|
||||||
session,
|
|
||||||
source_id="geizhals",
|
|
||||||
external_category_key="nb",
|
|
||||||
product_category_id="notebook",
|
|
||||||
name="Notebooks",
|
|
||||||
source_url="https://geizhals.de/?cat=nb",
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def _seed_source(session: Session) -> None:
|
|
||||||
_upsert_source(session, "manual", "Manual entry", "manual")
|
|
||||||
_upsert_source(session, "geizhals", "Geizhals", "geizhals")
|
|
||||||
|
|
||||||
|
|
||||||
def _upsert_source(session: Session, source_id: str, name: str, adapter_type: str) -> None:
|
|
||||||
existing = session.get(SourceRecord, source_id)
|
|
||||||
if existing is None:
|
|
||||||
session.add(SourceRecord(id=source_id, name=name, adapter_type=adapter_type))
|
|
||||||
return
|
|
||||||
if existing.adapter_type != adapter_type:
|
|
||||||
raise SystemExit(f"bootstrap conflict: source '{source_id}' has a different adapter_type")
|
|
||||||
|
|
||||||
|
|
||||||
def _seed_scoring_model(session: Session) -> None:
|
|
||||||
definition = json.loads(RULES.read_text(encoding="utf-8"))
|
|
||||||
model = parse_scoring_model(definition)
|
|
||||||
existing = session.get(ScoringModelRecord, model.version)
|
|
||||||
if existing is None:
|
|
||||||
session.add(
|
|
||||||
ScoringModelRecord(
|
|
||||||
version=model.version,
|
|
||||||
algorithm_id=model.algorithm_id,
|
|
||||||
definition=model.definition,
|
|
||||||
definition_hash=model.definition_hash,
|
|
||||||
created_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return
|
|
||||||
if existing.definition_hash != model.definition_hash:
|
|
||||||
raise SystemExit(
|
|
||||||
f"bootstrap conflict: scoring model {model.version} hash "
|
|
||||||
f"{existing.definition_hash} != {model.definition_hash}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
session = SessionLocal()
|
|
||||||
try:
|
|
||||||
run_bootstrap(session)
|
|
||||||
print("bootstrap ok")
|
|
||||||
finally:
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
@ -1,138 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent.parent
|
|
||||||
PROFILE = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
|
|
||||||
FIXTURE = json.loads((ROOT / "fixtures" / "manual-demo.json").read_text(encoding="utf-8"))
|
|
||||||
BASE = "http://127.0.0.1:8000"
|
|
||||||
AS_OF = "2026-10-03T12:00:00Z"
|
|
||||||
|
|
||||||
DEFINITIONS = [
|
|
||||||
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0, "description": "RAM in GB"},
|
|
||||||
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0, "description": "Storage in GB"},
|
|
||||||
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0, "description": "Weight in kg"},
|
|
||||||
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual battery score"},
|
|
||||||
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual display score"},
|
|
||||||
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual CPU score"},
|
|
||||||
{"key": "ports", "value_type": "string_list", "description": "Ports"},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def request(method: str, path: str, body=None, expected=(200, 201, 409)):
|
|
||||||
data = None if body is None else json.dumps(body, default=str).encode()
|
|
||||||
req = urllib.request.Request(
|
|
||||||
f"{BASE}{path}",
|
|
||||||
data=data,
|
|
||||||
method=method,
|
|
||||||
headers={"Content-Type": "application/json"} if body is not None else {},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(req) as response:
|
|
||||||
payload = json.loads(response.read().decode())
|
|
||||||
if response.status not in expected:
|
|
||||||
raise SystemExit(f"{method} {path} -> {response.status}: {payload}")
|
|
||||||
return response.status, payload
|
|
||||||
except urllib.error.HTTPError as exc:
|
|
||||||
payload = json.loads(exc.read().decode())
|
|
||||||
if exc.code not in expected:
|
|
||||||
raise SystemExit(f"{method} {path} -> {exc.code}: {payload}") from exc
|
|
||||||
return exc.code, payload
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
for definition in DEFINITIONS:
|
|
||||||
request("POST", "/api/v1/attribute-definitions", definition)
|
|
||||||
request("POST", "/api/v1/profiles", PROFILE)
|
|
||||||
|
|
||||||
product = FIXTURE["products"][0]
|
|
||||||
request(
|
|
||||||
"POST",
|
|
||||||
"/api/v1/products",
|
|
||||||
{
|
|
||||||
"id": product["id"],
|
|
||||||
"category": product["category"],
|
|
||||||
"brand": product["brand"],
|
|
||||||
"name": product["name"],
|
|
||||||
"model": product.get("model"),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
for variant in product["variants"]:
|
|
||||||
request(
|
|
||||||
"POST",
|
|
||||||
f"/api/v1/products/{product['id']}/variants",
|
|
||||||
{
|
|
||||||
"id": variant["id"],
|
|
||||||
"name": variant["name"],
|
|
||||||
"manufacturer_sku": variant.get("manufacturer_sku"),
|
|
||||||
"identifiers": variant.get("identifiers") or {},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
for attribute in variant["attributes"]:
|
|
||||||
request("POST", f"/api/v1/variants/{variant['id']}/attributes", {**attribute, "source_id": "manual"})
|
|
||||||
|
|
||||||
for offer in FIXTURE["offers"]:
|
|
||||||
request(
|
|
||||||
"POST",
|
|
||||||
f"/api/v1/variants/{offer['variant_id']}/offers",
|
|
||||||
{
|
|
||||||
"id": offer["id"],
|
|
||||||
"source_id": "manual",
|
|
||||||
"external_id": offer["external_id"],
|
|
||||||
"seller": offer["seller"],
|
|
||||||
"url": offer.get("url"),
|
|
||||||
"market": offer["market"],
|
|
||||||
"currency": offer["currency"],
|
|
||||||
"condition": offer["condition"],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
for price in FIXTURE["prices"]:
|
|
||||||
offer = next(item for item in FIXTURE["offers"] if item["external_id"] == price["external_offer_id"])
|
|
||||||
request(
|
|
||||||
"POST",
|
|
||||||
f"/api/v1/offers/{offer['id']}/prices",
|
|
||||||
{
|
|
||||||
"id": price["id"],
|
|
||||||
"source_id": "manual",
|
|
||||||
"external_observation_id": price["external_observation_id"],
|
|
||||||
"observed_at": price["observed_at"],
|
|
||||||
"item_price": price["item_price"],
|
|
||||||
"shipping_price": price["shipping_price"],
|
|
||||||
"currency": price["currency"],
|
|
||||||
"availability": price["availability"],
|
|
||||||
"raw_payload": price["raw_payload"],
|
|
||||||
"confidence": price["confidence"],
|
|
||||||
},
|
|
||||||
expected=(200, 201),
|
|
||||||
)
|
|
||||||
|
|
||||||
cases = {
|
|
||||||
"A": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-333333333333"),
|
|
||||||
"B": ("22222222-2222-4222-8222-22222222222b", "33333333-3333-4333-8333-33333333333b"),
|
|
||||||
"C": ("22222222-2222-4222-8222-22222222222c", "33333333-3333-4333-8333-33333333333c"),
|
|
||||||
"D": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-33333333333d"),
|
|
||||||
}
|
|
||||||
for label, (variant_id, offer_id) in cases.items():
|
|
||||||
status, result = request(
|
|
||||||
"POST",
|
|
||||||
f"/api/v1/evaluate/{variant_id}",
|
|
||||||
{
|
|
||||||
"profile_id": "mobile-development-notebook",
|
|
||||||
"profile_version": 1,
|
|
||||||
"offer_id": offer_id,
|
|
||||||
"as_of": AS_OF,
|
|
||||||
},
|
|
||||||
expected=(201,),
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
f"case {label}: {status} overall={result['overall_eligibility']} "
|
|
||||||
f"product={result['product_score']} price={result['price_score']} "
|
|
||||||
f"deal={result['deal_score']} class={result['classification']}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
"""Notebook-pilot metadata only. Not part of the core schema or bootstrap."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.discovery.definitions import get_definition_by_key, upsert_definition
|
|
||||||
from app.persistence.models import AttributeDefinitionRecord
|
|
||||||
from app.services.catalog_service import ensure_product_category, ensure_source_category
|
|
||||||
|
|
||||||
|
|
||||||
NOTEBOOK_ATTRIBUTE_SEEDS = (
|
|
||||||
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
|
||||||
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
|
|
||||||
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
|
|
||||||
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
|
||||||
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
|
||||||
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
|
|
||||||
)
|
|
||||||
|
|
||||||
NOTEBOOK_CRITERION_SEEDS = (
|
|
||||||
{
|
|
||||||
"key": "ram_gb",
|
|
||||||
"origin": "SOURCE",
|
|
||||||
"source": "geizhals",
|
|
||||||
"product_category_id": "notebook",
|
|
||||||
"label": "Arbeitsspeicher",
|
|
||||||
"value_type": "number",
|
|
||||||
"unit": "GB",
|
|
||||||
"semantic": "working_memory",
|
|
||||||
"category": "notebook",
|
|
||||||
"scoring_attribute_key": "ram_gb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "storage_gb",
|
|
||||||
"origin": "SOURCE",
|
|
||||||
"source": "geizhals",
|
|
||||||
"product_category_id": "notebook",
|
|
||||||
"label": "SSD",
|
|
||||||
"value_type": "number",
|
|
||||||
"unit": "GB",
|
|
||||||
"semantic": "storage",
|
|
||||||
"category": "notebook",
|
|
||||||
"scoring_attribute_key": "storage_gb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "weight_kg",
|
|
||||||
"origin": "SOURCE",
|
|
||||||
"source": "geizhals",
|
|
||||||
"product_category_id": "notebook",
|
|
||||||
"label": "Gewicht",
|
|
||||||
"value_type": "number",
|
|
||||||
"unit": "kg",
|
|
||||||
"semantic": "weight",
|
|
||||||
"category": "notebook",
|
|
||||||
"scoring_attribute_key": "weight_kg",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "price_eur",
|
|
||||||
"origin": "SOURCE",
|
|
||||||
"source": "geizhals",
|
|
||||||
"product_category_id": "notebook",
|
|
||||||
"label": "Preis",
|
|
||||||
"value_type": "number",
|
|
||||||
"unit": "EUR",
|
|
||||||
"semantic": "price",
|
|
||||||
"category": "notebook",
|
|
||||||
"scoring_attribute_key": "price_eur",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def seed_notebook_pilot(session: Session) -> None:
|
|
||||||
ensure_product_category(session, "notebook", "Notebooks")
|
|
||||||
ensure_source_category(
|
|
||||||
session,
|
|
||||||
source_id="geizhals",
|
|
||||||
external_category_key="nb",
|
|
||||||
product_category_id="notebook",
|
|
||||||
name="Notebooks",
|
|
||||||
source_url="https://geizhals.de/?cat=nb",
|
|
||||||
)
|
|
||||||
for item in NOTEBOOK_ATTRIBUTE_SEEDS:
|
|
||||||
if session.get(AttributeDefinitionRecord, item["key"]) is None:
|
|
||||||
session.add(AttributeDefinitionRecord(**item, description=None))
|
|
||||||
for item in NOTEBOOK_CRITERION_SEEDS:
|
|
||||||
if get_definition_by_key(session, item["key"]) is None:
|
|
||||||
upsert_definition(session, item)
|
|
||||||
session.flush()
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user