Kansho/backend/detect_pipeline.py

"""Student-first detect pipeline with optional teacher comparison.

Cues stay local. OpenRouter/GLiNER never create identities. Compact counters
carry no plaintext labels.
"""
from __future__ import annotations

import uuid
from dataclasses import dataclass, field

from detect_learning import (
    MODE_LEARNING_MANUAL,
    MODE_STUDENT,
    _CUE_BARE,
    _DECISION_ASK,
    _DECISION_IDENTITY,
    _DECISION_NOT,
    _align_user_span,
    _mention_fields,
    _resolve_user_span,
    _token_for_label,
    _word_matches,
    apply_learned_decisions,
    auto_resolve_ambiguous,
    cue_for_mention,
    decision_for_mention,
    get_cue_decision,
    get_detect_operating_mode,
    get_local_ner_settings,
    get_sense,
    is_bypass_mode,
    is_learning_mode,
    learning_stage,
    merge_gliner_candidates,
    record_cue,
    supplement_kinship_candidates,
)
from entity_detect import DetectError, detect_personal_egress
from identity_store import ENTITY_TYPES, is_maskable_label

KIND_DIFF = "diff"
KIND_MANUAL = "manual"
KIND_CONFIRM_REMOTE = "confirm_remote"

PROTECTING_TYPES = set(ENTITY_TYPES)
FOOD_TYPES = frozenset(
    {"FOOD", "DISH", "MEAL", "CUISINE", "FOODSTUFF", "INGREDIENT", "food", "dish"}
)
TYPE_MAP = {
    "person": "PERSON",
    "place": "PLACE",
    "org": "ORG",
    "organization": "ORG",
    "project": "PROJECT",
    "food": "FOOD",
    "dish": "DISH",
}


@dataclass
class MentionSpan:
    text: str
    start: int
    end: int
    entity_type: str
    protection: str
    source: str
    confirmed_cue: bool = False
    cue: str = _CUE_BARE

    def key(self) -> tuple[int, int, str]:
        return (self.start, self.end, self.text.casefold())


@dataclass
class StudentResult:
    mappings: list[dict] = field(default_factory=list)
    candidates: list[dict] = field(default_factory=list)
    spans: list[MentionSpan] = field(default_factory=list)
    gliner_missing: bool = False


@dataclass
class TeacherResult:
    spans: list[MentionSpan] = field(default_factory=list)
    mappings: list[dict] = field(default_factory=list)
    ok: bool = True
    error: str = ""


@dataclass
class PipelineResult:
    mappings: list[dict]
    candidates: list[dict]
    kind: str = KIND_DIFF
    need_review: bool = False
    need_remote_confirm: bool = False
    counters: dict[str, int] = field(default_factory=dict)
    mode: str = ""


def empty_counters() -> dict[str, int]:
    return {
        "diff_agree": 0,
        "diff_only_student": 0,
        "diff_only_teacher": 0,
        "diff_type_conflict": 0,
        "diff_new_identity": 0,
        "diff_cue_reinforced": 0,
    }


def map_entity_type(raw: str) -> str:
    token = (raw or "").strip()
    if token.upper() in PROTECTING_TYPES or token.upper() in {item.upper() for item in FOOD_TYPES}:
        mapped = TYPE_MAP.get(token.lower())
        return mapped or token.upper()
    return TYPE_MAP.get(token.lower()) or token.upper()


def protects(entity_type: str) -> bool:
    kind = map_entity_type(entity_type)
    if kind in FOOD_TYPES or kind.upper() in {item.upper() for item in FOOD_TYPES}:
        return False
    return kind in PROTECTING_TYPES


def types_equivalent(left: str, right: str) -> bool:
    a, b = map_entity_type(left), map_entity_type(right)
    if a in FOOD_TYPES or b in FOOD_TYPES or a in {"FOOD", "DISH"} or b in {"FOOD", "DISH"}:
        return a == b
    if protects(a) and protects(b):
        return True
    return a == b


def _contains(outer: MentionSpan, inner: MentionSpan) -> bool:
    return outer.start <= inner.start and inner.end <= outer.end and outer.end > outer.start


def same_mention(left: MentionSpan, right: MentionSpan) -> bool:
    if left.text.casefold() == right.text.casefold() and left.start == right.start and left.end == right.end:
        return True
    if _contains(left, right) or _contains(right, left):
        return True
    return False


def _collapse_same_protection(spans: list[MentionSpan]) -> list[MentionSpan]:
    if not spans:
        return []
    remaining = list(spans)
    chosen: list[MentionSpan] = []
    while remaining:
        current = remaining.pop(0)
        cluster = [current]
        rest = []
        for item in remaining:
            if same_mention(current, item):
                cluster.append(item)
            else:
                rest.append(item)
        remaining = rest
        protections = {item.protection for item in cluster}
        if len(protections) > 1:
            chosen.extend(cluster)
            continue
        chosen.append(max(cluster, key=lambda item: (item.end - item.start, -item.start)))
    return chosen


def _user_span_for_mapping(mapping: dict, user_body: str, rendered: str = "") -> tuple[int, int] | None:
    return _resolve_user_span(
        user_body,
        mapping.get("start"),
        mapping.get("end"),
        mapping.get("local_label") or "",
        rendered,
    )


def _mapping_from_user_span(
    mappings: list[dict],
    label: str,
    user_body: str,
    start: int,
    end: int,
    entity_type: str,
    rendered: str,
) -> dict:
    aligned = _align_user_span(rendered, user_body, start, end)
    kind = entity_type if entity_type in ENTITY_TYPES else "PERSON"
    return {
        "token": _token_for_label(mappings, label, kind),
        "local_label": label,
        "canonical_label": label,
        "demask_label": label,
        "entity_type": kind,
        "source": "request_local",
        "start": aligned[0],
        "end": aligned[1],
        "aliases": [],
        "labels": [label],
    }


def run_student_path(profile_id: str, user_body: str, rendered: str = "") -> StudentResult:
    mappings: list[dict] = []
    candidates: list[dict] = []
    spans: list[MentionSpan] = []
    text = user_body or ""
    for match in _word_matches(text):
        label = match.group(0)
        if not is_maskable_label(label):
            continue
        start, end = match.start(), match.end()
        verdict = decision_for_mention(profile_id, label, text, start, end, rendered)
        cue = cue_for_mention(text, start, end, label, rendered)
        if verdict == _DECISION_IDENTITY:
            mappings.append(_mapping_from_user_span(mappings, label, text, start, end, "PERSON", rendered))
            spans.append(
                MentionSpan(label, start, end, "PERSON", _DECISION_IDENTITY, "cue", True, cue)
            )
        elif verdict == _DECISION_NOT:
            spans.append(
                MentionSpan(label, start, end, "FOOD", _DECISION_NOT, "cue", True, cue)
            )
        elif get_sense(profile_id, label)["ambiguous"]:
            aligned = _align_user_span(rendered, text, start, end)
            candidates.append(
                {
                    "id": str(uuid.uuid4()),
                    "text": label,
                    "entity_type": "PERSON",
                    "start": aligned[0],
                    "end": aligned[1],
                    **_mention_fields(text, start, end, label, rendered),
                    "ambiguous": True,
                }
            )
    candidates, mappings = supplement_kinship_candidates(
        profile_id, candidates, mappings, text, rendered
    )
    candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
    bound = {(span.start, span.end, span.text.casefold()) for span in spans}
    for item in candidates:
        resolved = _resolve_user_span(
            text,
            item.get("user_start", item.get("start")),
            item.get("user_end", item.get("end")),
            item.get("text") or "",
            rendered,
        )
        if resolved:
            bound.add((resolved[0], resolved[1], (item.get("text") or "").casefold()))
    settings = get_local_ner_settings()
    gliner_missing = False
    if settings.get("enabled"):
        from gliner_client import extract_entities

        raw = extract_entities(text, url=settings.get("url"))
        if raw is None:
            gliner_missing = True
        else:
            usable = []
            for item in raw:
                try:
                    start, end = int(item["start"]), int(item["end"])
                except (KeyError, TypeError, ValueError):
                    continue
                label = (item.get("text") or "").casefold()
                if any(
                    not (end <= left or start >= right) and folded == label
                    for left, right, folded in bound
                ) or any(left <= start and end <= right for left, right, _folded in bound):
                    continue
                usable.append(item)
            candidates, mappings = merge_gliner_candidates(
                profile_id, candidates, mappings, text, rendered, usable
            )
    candidates, mappings = apply_learned_decisions(profile_id, candidates, mappings, text, rendered)
    for item in candidates:
        resolved = _resolve_user_span(
            text,
            item.get("user_start", item.get("start")),
            item.get("user_end", item.get("end")),
            item.get("text") or "",
            rendered,
        )
        if not resolved:
            continue
        suggested = item.get("suggested") or _DECISION_IDENTITY
        kind = item.get("entity_type") or "PERSON"
        protection = _DECISION_NOT if suggested == _DECISION_NOT else _DECISION_IDENTITY
        entity = "FOOD" if protection == _DECISION_NOT else kind
        spans.append(
            MentionSpan(
                item.get("text") or "",
                resolved[0],
                resolved[1],
                entity,
                protection,
                "candidate",
                False,
                item.get("cue") or _CUE_BARE,
            )
        )
    return StudentResult(
        mappings=mappings,
        candidates=candidates,
        spans=_collapse_same_protection(spans),
        gliner_missing=gliner_missing,
    )


def blank_cue_safe_forms(profile_id: str, user_body: str, rendered: str = "") -> str:
    chars = list(user_body or "")
    for match in _word_matches(user_body or ""):
        label = match.group(0)
        if not is_maskable_label(label):
            continue
        verdict = decision_for_mention(
            profile_id, label, user_body, match.start(), match.end(), rendered
        )
        if verdict not in {_DECISION_IDENTITY, _DECISION_NOT}:
            continue
        for index in range(match.start(), match.end()):
            chars[index] = " "
    return "".join(chars)


def _teacher_span(label: str, start: int, end: int, entity_type: str) -> MentionSpan:
    kind = map_entity_type(entity_type)
    protection = _DECISION_IDENTITY if protects(kind) else _DECISION_NOT
    return MentionSpan(label, start, end, kind, protection, "teacher", False)


def run_teacher(
    profile_id: str,
    user_body: str,
    *,
    stage: int = 1,
    rendered: str = "",
) -> TeacherResult:
    source = user_body or ""
    if stage >= 2:
        source = blank_cue_safe_forms(profile_id, user_body, rendered)
    try:
        outcome = detect_personal_egress(profile_id, source)
    except DetectError as exc:
        return TeacherResult(ok=False, error=exc.code)
    spans: list[MentionSpan] = []
    for mapping in outcome.mappings or []:
        label = mapping.get("local_label") or ""
        start, end = mapping.get("start"), mapping.get("end")
        resolved = None
        if start is not None and end is not None:
            left, right = int(start), int(end)
            if 0 <= left < right <= len(user_body) and user_body[left:right].casefold() == label.casefold():
                resolved = (left, right)
        if not resolved:
            resolved = _resolve_user_span(user_body, start, end, label, user_body)
        if not resolved:
            continue
        spans.append(
            _teacher_span(label, resolved[0], resolved[1], mapping.get("entity_type") or "PERSON")
        )
    omitted = list((outcome.stats.omitted_non_identity_types or []) if outcome.stats else [])
    for kind in omitted:
        if map_entity_type(kind) in FOOD_TYPES or kind.upper() in {"FOOD", "DISH"}:
            continue
    return TeacherResult(spans=_collapse_same_protection(spans), mappings=list(outcome.mappings or []), ok=True)


def _candidate_from_span(
    span: MentionSpan,
    user_body: str,
    rendered: str,
    *,
    suggested: str,
    reason: str,
) -> dict:
    aligned = _align_user_span(rendered, user_body, span.start, span.end)
    kind = span.entity_type if span.entity_type in ENTITY_TYPES else "PERSON"
    return {
        "id": str(uuid.uuid4()),
        "text": span.text,
        "entity_type": kind,
        "start": aligned[0],
        "end": aligned[1],
        **_mention_fields(user_body, span.start, span.end, span.text, rendered),
        "ambiguous": False,
        "suggested": suggested,
        "reason": reason,
    }


def compare_mentions(
    profile_id: str,
    user_body: str,
    student_spans: list[MentionSpan],
    teacher_spans: list[MentionSpan],
    rendered: str = "",
) -> tuple[list[dict], dict[str, int]]:
    counters = empty_counters()
    review: list[dict] = []
    used_teacher: set[int] = set()
    student_list = _collapse_same_protection(student_spans)
    teacher_list = _collapse_same_protection(teacher_spans)

    def take_teacher(index: int) -> MentionSpan:
        used_teacher.add(index)
        return teacher_list[index]

    for student in student_list:
        match_index = next(
            (
                index
                for index, teacher in enumerate(teacher_list)
                if index not in used_teacher and same_mention(student, teacher)
            ),
            None,
        )
        if match_index is None:
            counters["diff_only_student"] += 1
            if student.confirmed_cue:
                continue
            if student.protection == _DECISION_IDENTITY:
                counters["diff_new_identity"] += 1
                review.append(
                    _candidate_from_span(
                        student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_student"
                    )
                )
            continue
        teacher = take_teacher(match_index)
        if student.protection != teacher.protection or (
            protects(student.entity_type) != protects(teacher.entity_type)
        ):
            counters["diff_type_conflict"] += 1
            if student.confirmed_cue:
                continue
            review.append(
                _candidate_from_span(
                    student,
                    user_body,
                    rendered,
                    suggested=student.protection,
                    reason="type_conflict",
                )
            )
            continue
        if types_equivalent(student.entity_type, teacher.entity_type) or (
            student.protection == teacher.protection
        ):
            counters["diff_agree"] += 1
            if student.confirmed_cue:
                record_cue(profile_id, student.text, student.cue, student.protection)
                counters["diff_cue_reinforced"] += 1
            elif student.protection == _DECISION_IDENTITY:
                learned = get_cue_decision(profile_id, student.text, student.cue)
                if learned == _DECISION_IDENTITY:
                    record_cue(profile_id, student.text, student.cue, _DECISION_IDENTITY)
                    counters["diff_cue_reinforced"] += 1
                else:
                    counters["diff_new_identity"] += 1
                    review.append(
                        _candidate_from_span(
                            student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="new_identity"
                        )
                    )
    for index, teacher in enumerate(teacher_list):
        if index in used_teacher:
            continue
        counters["diff_only_teacher"] += 1
        cue = cue_for_mention(user_body, teacher.start, teacher.end, teacher.text, rendered)
        learned = get_cue_decision(profile_id, teacher.text, cue)
        if learned:
            continue
        if teacher.protection == _DECISION_IDENTITY:
            counters["diff_new_identity"] += 1
            review.append(
                _candidate_from_span(
                    teacher, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_teacher"
                )
            )
    return review, counters


def _remote_confirm_candidate(user_body: str) -> dict:
    snippet = (user_body or "")[:80]
    return {
        "id": str(uuid.uuid4()),
        "text": "Klartext-Detect",
        "entity_type": "PERSON",
        "start": 0,
        "end": 0,
        "excerpt": snippet,
        "highlight_start": None,
        "highlight_end": None,
        "user_start": None,
        "user_end": None,
        "cue": _CUE_BARE,
        "ambiguous": False,
        "suggested": _DECISION_NOT,
        "reason": KIND_CONFIRM_REMOTE,
    }


def prepare_detect_pipeline(profile_id: str, user_body: str, rendered: str = "") -> PipelineResult:
    mode = get_detect_operating_mode()
    counters = empty_counters()
    if is_bypass_mode(mode):
        return PipelineResult(mappings=[], candidates=[], kind=KIND_DIFF, mode=mode, counters=counters)
    student = run_student_path(profile_id, user_body, rendered)
    if mode == MODE_STUDENT:
        if student.gliner_missing:
            return PipelineResult(
                mappings=student.mappings,
                candidates=[_remote_confirm_candidate(user_body)],
                kind=KIND_CONFIRM_REMOTE,
                need_review=True,
                need_remote_confirm=True,
                counters=counters,
                mode=mode,
            )
        return PipelineResult(
            mappings=student.mappings,
            candidates=[],
            kind=KIND_DIFF,
            counters=counters,
            mode=mode,
        )
    if mode == MODE_LEARNING_MANUAL or student.gliner_missing:
        return PipelineResult(
            mappings=student.mappings,
            candidates=student.candidates,
            kind=KIND_MANUAL,
            need_review=bool(student.candidates),
            counters=counters,
            mode=mode,
        )
    teacher = run_teacher(profile_id, user_body, stage=learning_stage(mode), rendered=rendered)
    if not teacher.ok:
        return PipelineResult(
            mappings=student.mappings,
            candidates=student.candidates,
            kind=KIND_MANUAL,
            need_review=bool(student.candidates),
            counters=counters,
            mode=mode,
        )
    review, counters = compare_mentions(
        profile_id, user_body, student.spans, teacher.spans, rendered
    )
    merged = list(student.mappings)
    seen = {
        (
            item.get("start"),
            item.get("end"),
            (item.get("local_label") or "").casefold(),
        )
        for item in merged
    }
    for mapping in teacher.mappings:
        key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
        if key in seen:
            continue
        label = mapping.get("local_label") or ""
        resolved = _resolve_user_span(
            user_body, mapping.get("start"), mapping.get("end"), label, user_body
        )
        if resolved:
            cue = cue_for_mention(user_body, resolved[0], resolved[1], label, rendered)
            if get_cue_decision(profile_id, label, cue) == _DECISION_NOT:
                continue
        merged.append(mapping)
        seen.add(key)
    return PipelineResult(
        mappings=merged,
        candidates=review,
        kind=KIND_DIFF,
        need_review=bool(review),
        counters=counters,
        mode=mode,
    )


def compact_diff_counters(counters: dict[str, int] | None) -> dict[str, int]:
    payload = empty_counters()
    for key in payload:
        payload[key] = int((counters or {}).get(key) or 0)
    return payload


def apply_remote_confirm(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
    """After the user allows plaintext detect, run teacher on the user line."""
    approved = any((item.get("decision") or "").strip() == _DECISION_IDENTITY for item in decisions)
    if not approved:
        raise DetectError(
            "remote_detect_declined",
            "Klartext-Detect wurde nicht bestätigt.",
            403,
        )
    user_body = pending.get("user_body") or ""
    teacher = run_teacher(profile_id, user_body, stage=1)
    if not teacher.ok:
        raise DetectError(teacher.error or "detect_provider_unavailable", "Lehrer-Detect ist nicht erreichbar.")
    merged = list(pending.get("mappings") or [])
    seen = {
        (item.get("start"), item.get("end"), (item.get("local_label") or "").casefold())
        for item in merged
    }
    for mapping in teacher.mappings:
        key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
        if key in seen:
            continue
        merged.append(mapping)
        seen.add(key)
    return merged