mitai-jinkendo/backend/data_layer/food_mapping.py

"""FDDB → food_catalog mapping: normalize, lookup (user then global), learn, apply."""
from __future__ import annotations

import re
import unicodedata
from datetime import date, timedelta
from typing import Any

LEADING_QTY_RE = re.compile(
    r"^\s*\d+(?:[.,]\d+)?\s*(?:g|kg|ml|l|stück|stk|st\.?|portion(?:en)?)\b[\s,.:\-–]*",
    re.IGNORECASE,
)
TRAILING_QTY_RE = re.compile(
    r"[\s,.:\-–]+\d+(?:[.,]\d+)?\s*(?:g|kg|ml|l)\s*$",
    re.IGNORECASE,
)
MULTISPACE_RE = re.compile(r"\s+")
DECIMAL_IN_NAME_RE = re.compile(r"(\d),(\d)")


def strip_leading_quantity(raw: str | None) -> str:
    if not raw:
        return ""
    s = unicodedata.normalize("NFKC", str(raw)).strip().strip('"').strip("'")
    s = s.lstrip("!")
    s = LEADING_QTY_RE.sub("", s)
    return MULTISPACE_RE.sub(" ", s).strip()


def normalize_food_name(raw: str | None) -> str:
    s = strip_leading_quantity(raw)
    if not s:
        return ""
    s = TRAILING_QTY_RE.sub("", s)
    s = s.replace("-", " ")
    s = DECIMAL_IN_NAME_RE.sub(r"\1.\2", s)
    return MULTISPACE_RE.sub(" ", s).strip().lower()


def item_name_keys(raw: str | None, stored_norm: str | None = None) -> set[str]:
    """Compare diary names to list names despite stored quantity prefixes."""
    keys: set[str] = set()
    for v in (raw, stored_norm):
        if not v:
            continue
        text = str(v).strip()
        if text:
            keys.add(text.lower())
        norm = normalize_food_name(text)
        if norm:
            keys.add(norm)
    return keys


def names_match_list(raw: str | None, stored_norm: str | None, list_norm: str | None) -> bool:
    target = normalize_food_name(list_norm) if list_norm else ""
    if not target:
        return False
    return target in item_name_keys(raw, stored_norm)


_LIST_COMMA_RE = re.compile(r",(?!\s*\d)")


def primary_search_query(query: str | None) -> str:
    """Use the name before a list-comma, but keep decimal commas (9,5 %)."""
    q = (query or "").strip()
    if not q:
        return ""
    return _LIST_COMMA_RE.split(q, maxsplit=1)[0].strip()


def merge_unmapped_rows(rows: list[dict]) -> list[dict]:
    merged: dict[str, dict] = {}
    for row in rows:
        raw = row.get("source_name_raw") or ""
        key = normalize_food_name(raw) or row.get("source_name_normalized") or raw.lower()
        if not key:
            continue
        display = strip_leading_quantity(raw) or raw
        count = int(row.get("count") or 0)
        if key not in merged:
            item = dict(row)
            item["source_name_normalized"] = key
            item["source_name_raw"] = display
            item["count"] = count
            item["variant_count"] = 1
            merged[key] = item
            continue
        cur = merged[key]
        cur["count"] = int(cur.get("count") or 0) + count
        cur["variant_count"] = int(cur.get("variant_count") or 1) + 1
        if display and (not cur.get("source_name_raw") or len(display) < len(cur["source_name_raw"])):
            cur["source_name_raw"] = display
        if row.get("matching_recipe_id") and not cur.get("matching_recipe_id"):
            cur["matching_recipe_id"] = row["matching_recipe_id"]
        if row.get("sample_quantity_raw") and not cur.get("sample_quantity_raw"):
            cur["sample_quantity_raw"] = row["sample_quantity_raw"]
        first, last = row.get("first_date"), row.get("last_date")
        if first and (not cur.get("first_date") or str(first) < str(cur["first_date"])):
            cur["first_date"] = first
        if last and (not cur.get("last_date") or str(last) > str(cur["last_date"])):
            cur["last_date"] = last
    return list(merged.values())


def as_iso_date(value: Any) -> str | None:
    if value is None or value == "":
        return None
    if hasattr(value, "isoformat"):
        return str(value.isoformat())[:10]
    text = str(value).strip()
    return text[:10] if len(text) >= 10 else None


def filter_unmapped_since(rows: list[dict], since_days: int, today: date | None = None) -> list[dict]:
    """Keep names last eaten in the window. Rows without last_date drop out (old recipe leftovers)."""
    days = int(since_days or 0)
    if days <= 0:
        return list(rows)
    cutoff = ((today or date.today()) - timedelta(days=days)).isoformat()
    out = []
    for row in rows:
        last = as_iso_date(row.get("last_date"))
        if last and last >= cutoff:
            out.append(row)
    return out


def fetch_unmapped_rows(cur, profile_id: str) -> list[dict[str, Any]]:
    """Diary lines without food/recipe plus list ingredients without mapping.

    Ingredient dates come from recipe.updated_at or last diary use of that list
    (items that already have recipe_id) — not a join of every recipe name to
    every diary row.
    """
    cur.execute(
        """
        SELECT source_name_normalized
        FROM food_name_mappings
        WHERE profile_id = %s
        """,
        (profile_id,),
    )
    mapped = {r["source_name_normalized"] for r in cur.fetchall()}
    cur.execute(
        """
        SELECT i.source_name_raw, i.source_name_normalized,
               COUNT(*) AS count, MIN(i.date) AS first_date, MAX(i.date) AS last_date,
               MIN(r.id::text) AS matching_recipe_id,
               MIN(i.quantity_raw) AS sample_quantity_raw
        FROM nutrition_items i
        LEFT JOIN food_recipes r
          ON r.profile_id = i.profile_id AND r.name_normalized = i.source_name_normalized
        WHERE i.profile_id = %s AND i.food_id IS NULL AND i.recipe_id IS NULL
        GROUP BY i.source_name_raw, i.source_name_normalized
        """,
        (profile_id,),
    )
    diary = [dict(r) | {"kind": "diary"} for r in cur.fetchall()]
    cur.execute(
        """
        SELECT id, name_raw, name_normalized
        FROM food_recipes
        WHERE profile_id = %s
        """,
        (profile_id,),
    )
    lists = [dict(r) for r in cur.fetchall()]
    for row in diary:
        keys = item_name_keys(row.get("source_name_raw"), row.get("source_name_normalized"))
        hit = next(
            (
                rec
                for rec in lists
                if rec.get("name_normalized") in keys
                or normalize_food_name(rec.get("name_raw")) in keys
            ),
            None,
        )
        if hit:
            row["matching_recipe_id"] = str(hit["id"])
            row["matching_recipe_name"] = hit["name_raw"]
    cur.execute(
        """
        SELECT i.source_name_raw, i.source_name_normalized,
               COUNT(*) AS count,
               MIN(COALESCE(u.last_used, r.updated_at::date, r.created_at::date)) AS first_date,
               MAX(COALESCE(u.last_used, r.updated_at::date, r.created_at::date)) AS last_date,
               MIN(i.quantity_raw) AS sample_quantity_raw
        FROM food_recipe_ingredients i
        JOIN food_recipes r ON r.id = i.recipe_id
        LEFT JOIN (
            SELECT recipe_id, MIN(date) AS first_used, MAX(date) AS last_used
            FROM nutrition_items
            WHERE profile_id = %s AND recipe_id IS NOT NULL
            GROUP BY recipe_id
        ) u ON u.recipe_id = r.id
        WHERE r.profile_id = %s
          AND NOT EXISTS (
            SELECT 1 FROM food_name_mappings m
            WHERE m.profile_id = r.profile_id
              AND m.source_name_normalized = i.source_name_normalized
          )
        GROUP BY i.source_name_raw, i.source_name_normalized
        """,
        (profile_id, profile_id),
    )
    ings = [dict(r) | {"kind": "recipe_ingredient"} for r in cur.fetchall()]
    out = []
    for row in merge_unmapped_rows(diary + ings):
        key = row.get("source_name_normalized") or normalize_food_name(row.get("source_name_raw"))
        if not key or key in mapped:
            continue
        row["last_date"] = as_iso_date(row.get("last_date"))
        row["first_date"] = as_iso_date(row.get("first_date"))
        out.append(row)
    return out


def list_unmapped_payload(
    cur,
    profile_id: str,
    since_days: int = 0,
    count_only: bool = False,
    meta: bool = False,
) -> Any:
    days = max(0, min(int(since_days or 0), 3650))
    out = fetch_unmapped_rows(cur, profile_id)
    items = sort_unmapped_rows(filter_unmapped_since(out, days) if days else out, days)
    if count_only:
        return {"count": len(items), "total": len(out), "since_days": days}
    if meta:
        return {"items": items, "count": len(items), "total": len(out), "since_days": days}
    return items


def sort_unmapped_rows(rows: list[dict], since_days: int = 0) -> list[dict]:
    rows = list(rows)
    if int(since_days or 0) > 0:
        rows.sort(
            key=lambda x: (
                as_iso_date(x.get("last_date")) or "",
                int(x.get("count") or 0),
            ),
            reverse=True,
        )
    else:
        rows.sort(key=lambda x: (-int(x.get("count") or 0), x.get("source_name_normalized") or ""))
    return rows


UNIT_ALIASES = {
    "g": "g", "gr": "g", "gramm": "g",
    "kg": "kg",
    "ml": "ml",
    "l": "l", "liter": "l", "lt": "l",
    "stück": "stück", "stk": "stück", "st": "stück", "st.": "stück", "pcs": "stück",
    "el": "el", "esslöffel": "el",
    "tl": "tl", "teelöffel": "tl",
    "prise": "prise",
    "scheibe": "scheibe",
    "portion": "portion", "portionen": "portion",
    "becher": "becher",
    "tasse": "tasse",
    "msp": "msp", "msp.": "msp",
}
MASS_VOLUME_TO_G = {"g": 1.0, "kg": 1000.0, "ml": 1.0, "l": 1000.0}
DEFAULT_UNIT_G = {"el": 15.0, "tl": 5.0, "prise": 0.3, "msp": 1.0}
COUNT_UNITS = frozenset({"stück", "scheibe", "portion", "becher", "tasse"})
UNIT_LABELS = {
    "g": "g", "kg": "kg", "ml": "ml", "l": "l",
    "el": "EL", "tl": "TL", "prise": "Prise", "msp": "Msp.",
    "stück": "Stück", "scheibe": "Scheibe", "portion": "Portion",
    "becher": "Becher", "tasse": "Tasse",
}
UNIT_ORDER = (
    "g", "ml", "el", "tl", "prise", "msp",
    "stück", "scheibe", "portion", "becher", "tasse", "kg", "l",
)
QTY_PARSE_RE = re.compile(
    r"^\s*(\d+(?:[.,]\d+)?)\s*([a-zA-ZäöüÄÖÜß.]+)?\s*$",
    re.IGNORECASE,
)


def _canon_unit(raw: str | None) -> str | None:
    if not raw:
        return None
    return UNIT_ALIASES.get(raw.strip().lower().rstrip("."))


def parse_quantity(raw: str | None, grams_per_unit: float | None = None) -> dict[str, Any]:
    empty = {"value": None, "unit": None, "quantity_g": None, "needs_unit_map": False}
    if raw is None or str(raw).strip() == "":
        return empty
    text = str(raw).strip().replace(",", ".")
    m = QTY_PARSE_RE.match(text)
    if not m:
        return empty
    value = float(m.group(1))
    unit = _canon_unit(m.group(2))
    if unit is None:
        return {"value": value, "unit": "g", "quantity_g": round(value, 3), "needs_unit_map": False}
    if unit in MASS_VOLUME_TO_G:
        return {
            "value": value,
            "unit": unit,
            "quantity_g": round(value * MASS_VOLUME_TO_G[unit], 3),
            "needs_unit_map": False,
        }
    factor = grams_per_unit if grams_per_unit is not None else DEFAULT_UNIT_G.get(unit)
    if factor is not None:
        return {
            "value": value,
            "unit": unit,
            "quantity_g": round(value * float(factor), 3),
            "needs_unit_map": unit in COUNT_UNITS,
        }
    return {"value": value, "unit": unit, "quantity_g": None, "needs_unit_map": True}


def parse_quantity_g(raw: str | None, grams_per_unit: float | None = None) -> float | None:
    return parse_quantity(raw, grams_per_unit)["quantity_g"]


def detect_quantity_unit(*texts: str | None) -> str | None:
    for text in texts:
        unit = parse_quantity(text).get("unit")
        if unit and unit not in MASS_VOLUME_TO_G:
            return unit
    return None


def format_quantity_raw(value: float | None, unit: str | None) -> str | None:
    if value is None:
        return None
    label = UNIT_LABELS.get(unit or "g", unit or "g")
    num = int(value) if float(value) == int(value) else value
    return f"{num} {label}".replace(".", ",")


def list_quantity_units() -> list[dict[str, Any]]:
    out = []
    for uid in UNIT_ORDER:
        default_g = MASS_VOLUME_TO_G.get(uid)
        if default_g is None:
            default_g = DEFAULT_UNIT_G.get(uid)
        out.append({
            "id": uid,
            "label": UNIT_LABELS[uid],
            "default_g": default_g,
            "needs_unit_map": uid in COUNT_UNITS,
        })
    return out


def resolve_quantity(
    *,
    quantity_raw: str | None = None,
    quantity_amount: float | None = None,
    source_unit: str | None = None,
    quantity_g: float | None = None,
    grams_per_unit: float | None = None,
) -> dict[str, Any]:
    """Amount + unit → quantity_g. Standard unit is grams; conversion uses mapping or defaults."""
    amount = None
    if quantity_amount not in (None, ""):
        try:
            amount = float(quantity_amount)
        except (TypeError, ValueError):
            amount = None
    explicit_g = None
    if quantity_g not in (None, ""):
        try:
            explicit_g = float(quantity_g)
        except (TypeError, ValueError):
            explicit_g = None
    if amount is not None:
        unit = _canon_unit(source_unit) or "g"
        raw = format_quantity_raw(amount, unit)
        parsed = parse_quantity(raw.replace(",", "."), grams_per_unit)
    elif quantity_raw:
        parsed = parse_quantity(quantity_raw, grams_per_unit)
    elif explicit_g is not None:
        parsed = {"value": explicit_g, "unit": "g", "quantity_g": explicit_g, "needs_unit_map": False}
    else:
        parsed = {"value": None, "unit": _canon_unit(source_unit), "quantity_g": None, "needs_unit_map": False}
    qty_g = parsed.get("quantity_g")
    if qty_g is None and explicit_g is not None:
        qty_g = explicit_g
    unit = parsed.get("unit")
    value = parsed.get("value")
    return {
        "quantity_amount": value,
        "source_unit": unit,
        "quantity_raw": format_quantity_raw(value, unit) or (str(quantity_raw).strip() if quantity_raw else None),
        "quantity_g": qty_g,
        "needs_unit_map": bool(parsed.get("needs_unit_map")),
    }


def get_food_mapping_with_cursor(
    cur,
    source_name: str,
    profile_id: str | None = None,
    source_system: str = "fddb",
) -> dict[str, Any] | None:
    norm = normalize_food_name(source_name)
    if not norm:
        return None
    if profile_id:
        cur.execute(
            """
            SELECT m.id AS mapping_id, m.food_id, m.profile_id, m.source,
                   m.grams_per_unit, m.source_unit,
                   f.bls_code, f.name_de, f.catalog_kind, f.owner_profile_id
            FROM food_name_mappings m
            JOIN food_catalog f ON f.id = m.food_id
            WHERE m.source_system = %s AND m.source_name_normalized = %s
              AND m.profile_id = %s
              AND f.is_active = true
              AND (f.owner_profile_id IS NULL OR f.owner_profile_id = %s)
            LIMIT 1
            """,
            (source_system, norm, profile_id, profile_id),
        )
        row = cur.fetchone()
        if row:
            return dict(row)
    cur.execute(
        """
        SELECT m.id AS mapping_id, m.food_id, m.profile_id, m.source,
               m.grams_per_unit, m.source_unit,
               f.bls_code, f.name_de, f.catalog_kind, f.owner_profile_id
        FROM food_name_mappings m
        JOIN food_catalog f ON f.id = m.food_id
        WHERE m.source_system = %s AND m.source_name_normalized = %s
          AND m.profile_id IS NULL
          AND f.is_active = true
          AND f.owner_profile_id IS NULL
        LIMIT 1
        """,
        (source_system, norm),
    )
    row = cur.fetchone()
    return dict(row) if row else None


def upsert_food_mapping(
    cur,
    *,
    source_name_raw: str,
    food_id: str,
    profile_id: str | None,
    source: str = "bulk",
    source_system: str = "fddb",
    grams_per_unit: float | None = None,
    source_unit: str | None = None,
) -> int:
    from data_layer.food_visibility import require_mapping_target

    require_mapping_target(cur, food_id, profile_id)
    norm = normalize_food_name(source_name_raw)
    if not norm:
        raise ValueError("Leerer Lebensmittelname")
    if profile_id:
        cur.execute(
            """
            SELECT id FROM food_name_mappings
            WHERE source_system = %s AND source_name_normalized = %s AND profile_id = %s
            """,
            (source_system, norm, profile_id),
        )
    else:
        cur.execute(
            """
            SELECT id FROM food_name_mappings
            WHERE source_system = %s AND source_name_normalized = %s AND profile_id IS NULL
            """,
            (source_system, norm),
        )
    existing = cur.fetchone()
    raw = source_name_raw.strip()
    unit = _canon_unit(source_unit) if source_unit else None
    if existing:
        cur.execute(
            """
            UPDATE food_name_mappings
            SET food_id = %s, source_name_raw = %s, source = %s,
                grams_per_unit = %s, source_unit = %s, updated_at = NOW()
            WHERE id = %s
            """,
            (food_id, raw, source, grams_per_unit, unit, existing["id"]),
        )
        return int(existing["id"])
    cur.execute(
        """
        INSERT INTO food_name_mappings
            (source_system, source_name_raw, source_name_normalized, food_id, profile_id,
             source, grams_per_unit, source_unit, updated_at)
        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW())
        RETURNING id
        """,
        (source_system, raw, norm, food_id, profile_id, source, grams_per_unit, unit),
    )
    return int(cur.fetchone()["id"])


def apply_quantities_to_items(cur, profile_id: str, source_name_normalized: str, grams_per_unit: float | None) -> int:
    if not grams_per_unit:
        return 0
    cur.execute(
        """
        SELECT id, quantity_raw, source_name_raw
        FROM nutrition_items
        WHERE profile_id = %s AND source_name_normalized = %s
        """,
        (profile_id, source_name_normalized),
    )
    n = 0
    for row in cur.fetchall():
        qty = parse_quantity_g(row.get("quantity_raw") or row.get("source_name_raw"), grams_per_unit)
        if qty is None:
            continue
        cur.execute(
            "UPDATE nutrition_items SET quantity_g = %s, updated_at = NOW() WHERE id = %s",
            (qty, row["id"]),
        )
        n += 1
    return n


def apply_mapping_to_items(cur, profile_id: str, source_name_normalized: str, food_id: str, mapping_id: int) -> int:
    origin = _value_origin_for_food(cur, food_id)
    cur.execute(
        """
        UPDATE nutrition_items
        SET food_id = %s, mapping_id = %s, value_origin = %s, updated_at = NOW()
        WHERE profile_id = %s AND source_name_normalized = %s
          AND recipe_id IS NULL
        """,
        (food_id, mapping_id, origin, profile_id, source_name_normalized),
    )
    n = cur.rowcount or 0
    cur.execute(
        """
        SELECT id, source_name_raw
        FROM nutrition_items
        WHERE profile_id = %s AND recipe_id IS NULL AND food_id IS NULL
        """,
        (profile_id,),
    )
    extra = [
        row["id"] for row in cur.fetchall()
        if normalize_food_name(row.get("source_name_raw")) == source_name_normalized
    ]
    if extra:
        cur.execute(
            """
            UPDATE nutrition_items
            SET food_id = %s, mapping_id = %s, value_origin = %s, updated_at = NOW()
            WHERE id = ANY(%s)
            """,
            (food_id, mapping_id, origin, extra),
        )
        n += cur.rowcount or 0
    return n


def clear_mapping_from_items(cur, profile_id: str, source_name_normalized: str) -> int:
    cur.execute(
        """
        UPDATE nutrition_items
        SET food_id = NULL, mapping_id = NULL, value_origin = 'fddb', updated_at = NOW()
        WHERE profile_id = %s AND source_name_normalized = %s
        """,
        (profile_id, source_name_normalized),
    )
    return cur.rowcount or 0


def _value_origin_for_food(cur, food_id: str) -> str:
    cur.execute("SELECT catalog_kind FROM food_catalog WHERE id = %s", (food_id,))
    row = cur.fetchone()
    if not row:
        return "fddb"
    kind = row["catalog_kind"]
    if kind == "official_bls":
        return "bls"
    return "manual_catalog"


_SEARCH_NUM_RE = re.compile(r"\d+(?:[.,]\d+)?")


def _catalog_must_tokens(query: str) -> list[list[str]]:
    """AND-groups for catalog search: first word plus each number (with 9,5/10 aliases)."""
    text = normalize_food_name(primary_search_query(query))
    nums = [m.replace(",", ".") for m in _SEARCH_NUM_RE.findall(text)]
    words = [t for t in re.split(r"[^a-z0-9äöüß]+", _SEARCH_NUM_RE.sub(" ", text)) if len(t) >= 2]
    groups: list[list[str]] = []
    if words:
        groups.append([words[0]])
    for num in nums[:3]:
        variants = {num, num.replace(".", ",")}
        try:
            value = float(num)
        except ValueError:
            value = None
        if value is not None and (abs(value - 10) <= 0.6 or abs(value - 9.5) <= 0.6):
            variants.update({"10", "9.5", "9,5"})
        groups.append(list(variants))
    return groups


def suggest_catalog_foods(cur, query: str, profile_id: str | None, limit: int = 8) -> list[dict]:
    q = (query or "").strip()
    if not q:
        return []
    primary = primary_search_query(q) or q
    like_full = f"%{q}%"
    like_primary = f"%{primary}%"
    prefix = f"{primary}%"
    norm = normalize_food_name(primary)
    extra_sql = ""
    extra_params: list[str] = []
    must = _catalog_must_tokens(q)
    if len(must) >= 2:
        parts = []
        for group in must:
            ors = " OR ".join(["name_de ILIKE %s"] * len(group))
            parts.append(f"({ors})")
            extra_params.extend(f"%{v}%" for v in group)
        extra_sql = " OR (" + " AND ".join(parts) + ")"
    cur.execute(
        f"""
        SELECT id, bls_code, name_de, name_en, catalog_kind, food_group, owner_profile_id
        FROM food_catalog
        WHERE is_active = true
          AND (
              owner_profile_id IS NULL
              OR owner_profile_id = %s
          )
          AND (
              name_de ILIKE %s OR name_de ILIKE %s
              OR COALESCE(name_en, '') ILIKE %s OR COALESCE(name_en, '') ILIKE %s
              OR COALESCE(bls_code, '') ILIKE %s
              OR lower(name_de) = %s
              {extra_sql}
          )
        ORDER BY
            CASE
                 WHEN lower(name_de) = %s THEN 0
                 WHEN name_de ILIKE %s THEN 1
                 WHEN COALESCE(bls_code, '') ILIKE %s THEN 2
                 ELSE 3 END,
            CASE catalog_kind
                 WHEN 'manual_user' THEN 0
                 WHEN 'manual_admin' THEN 1
                 ELSE 2 END,
            name_de
        LIMIT %s
        """,
        (
            profile_id,
            like_full, like_primary, like_full, like_primary, like_full, norm,
            *extra_params,
            norm, prefix, q, limit,
        ),
    )
    return [dict(r) for r in cur.fetchall()]