Put student-first detect learning ahead of the teacher call.
Identity-only senses no longer lock a new cue. Learning stages, journal-generate review, and additive docs follow the closed section 6. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
a65ea98b0c
commit
011cc9dd81
|
|
@ -1,12 +1,13 @@
|
|||
"""Transitional learning detect: senses grow from dialogue confirmations.
|
||||
|
||||
Not a global word list. Detect still does not auto-activate identities.
|
||||
Ambiguous spellings get a local passage call when a local detect provider exists.
|
||||
Stored passages are the context a later local GLiNER/detect can use for pre-decision.
|
||||
Student path: cues, kinship, homonym passage, then GLiNER. Teacher is optional.
|
||||
Uncertain mentions go as a mini-passage to a chosen LLM profile, not to Detect.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
|
@ -21,10 +22,31 @@ from identity_store import (
|
|||
normalize_label,
|
||||
)
|
||||
|
||||
MODE_STUDENT = "student"
|
||||
MODE_LEARNING_FULL = "learning_full"
|
||||
MODE_LEARNING_REDUCED = "learning_reduced"
|
||||
MODE_LEARNING_MANUAL = "learning_manual"
|
||||
MODE_BYPASS = "bypass"
|
||||
MODE_SEMANTIC = "semantic"
|
||||
MODE_LEARNING = "learning"
|
||||
MODES = (MODE_SEMANTIC, MODE_LEARNING)
|
||||
CANONICAL_MODES = (
|
||||
MODE_STUDENT,
|
||||
MODE_LEARNING_FULL,
|
||||
MODE_LEARNING_REDUCED,
|
||||
MODE_LEARNING_MANUAL,
|
||||
MODE_BYPASS,
|
||||
)
|
||||
MODE_ALIASES = {
|
||||
MODE_SEMANTIC: MODE_BYPASS,
|
||||
MODE_LEARNING: MODE_LEARNING_FULL,
|
||||
}
|
||||
MODES = CANONICAL_MODES + (MODE_SEMANTIC, MODE_LEARNING)
|
||||
LEARNING_MODES = {MODE_LEARNING_FULL, MODE_LEARNING_REDUCED, MODE_LEARNING_MANUAL}
|
||||
SETTING_KEY = "detect_operating_mode"
|
||||
GLINER_ENABLED_KEY = "gliner_enabled"
|
||||
GLINER_URL_KEY = "gliner_url"
|
||||
PASSAGE_PROFILE_KEY = "passage_profile_id"
|
||||
GLINER_DEFAULT_URL = "http://192.168.2.144:8090"
|
||||
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||
WINDOW = 80
|
||||
|
||||
|
|
@ -34,18 +56,32 @@ _DECISION_ASK = "ask"
|
|||
_CUE_BARE = "_bare"
|
||||
|
||||
|
||||
def canonicalize_detect_mode(mode: str | None) -> str | None:
|
||||
chosen = (mode or "").strip().lower()
|
||||
if not chosen:
|
||||
return None
|
||||
mapped = MODE_ALIASES.get(chosen, chosen)
|
||||
return mapped if mapped in CANONICAL_MODES else None
|
||||
|
||||
|
||||
def _default_detect_mode() -> str:
|
||||
if os.environ.get("KANSHO_FAKE_PROVIDER") or os.environ.get("KANSHO_FAKE_DETECT"):
|
||||
return MODE_BYPASS
|
||||
return MODE_LEARNING_FULL
|
||||
|
||||
|
||||
def get_detect_operating_mode() -> str:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
|
||||
)
|
||||
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
|
||||
return value if value in MODES else MODE_SEMANTIC
|
||||
stored = canonicalize_detect_mode((row or {}).get("value") or "")
|
||||
return stored or _default_detect_mode()
|
||||
|
||||
|
||||
def set_detect_operating_mode(mode: str) -> str:
|
||||
chosen = (mode or "").strip().lower()
|
||||
if chosen not in MODES:
|
||||
chosen = canonicalize_detect_mode(mode)
|
||||
if not chosen:
|
||||
raise ValueError("invalid_detect_operating_mode")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
|
|
@ -59,6 +95,88 @@ def set_detect_operating_mode(mode: str) -> str:
|
|||
return chosen
|
||||
|
||||
|
||||
def is_bypass_mode(mode: str | None = None) -> bool:
|
||||
return (mode or get_detect_operating_mode()) == MODE_BYPASS
|
||||
|
||||
|
||||
def is_learning_mode(mode: str | None = None) -> bool:
|
||||
return (mode or get_detect_operating_mode()) in LEARNING_MODES
|
||||
|
||||
|
||||
def learning_stage(mode: str | None = None) -> int:
|
||||
current = mode or get_detect_operating_mode()
|
||||
if current == MODE_LEARNING_REDUCED:
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _app_setting(key: str, default: str = "") -> str:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
|
||||
)
|
||||
if not row or row.get("value") is None:
|
||||
return default
|
||||
return str(row.get("value") or "").strip() or default
|
||||
|
||||
|
||||
def _put_app_setting(key: str, value: str) -> None:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO app_settings (key, value, updated)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
|
||||
""",
|
||||
(key, value),
|
||||
)
|
||||
|
||||
|
||||
def get_local_ner_settings() -> dict:
|
||||
from gliner_client import health, normalize_base_url
|
||||
from providers import is_local_url
|
||||
|
||||
enabled_raw = _app_setting(GLINER_ENABLED_KEY, "0").lower()
|
||||
enabled = enabled_raw in {"1", "true", "yes", "on"}
|
||||
url = normalize_base_url(_app_setting(GLINER_URL_KEY, GLINER_DEFAULT_URL) or GLINER_DEFAULT_URL)
|
||||
passage_profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
|
||||
ping = health(url) if enabled and is_local_url(url) else {"ok": False, "status": "off"}
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"url": url,
|
||||
"passage_profile_id": passage_profile_id,
|
||||
"ready": bool(enabled and ping.get("ok")),
|
||||
"health": ping,
|
||||
}
|
||||
|
||||
|
||||
def set_local_ner_settings(
|
||||
*,
|
||||
enabled: bool | None = None,
|
||||
url: str | None = None,
|
||||
passage_profile_id: str | None = None,
|
||||
) -> dict:
|
||||
from gliner_client import normalize_base_url
|
||||
from providers import is_local_url
|
||||
|
||||
if enabled is not None:
|
||||
_put_app_setting(GLINER_ENABLED_KEY, "1" if enabled else "0")
|
||||
if url is not None:
|
||||
chosen = normalize_base_url(url) if (url or "").strip() else GLINER_DEFAULT_URL
|
||||
if not is_local_url(chosen):
|
||||
raise ValueError("gliner_url_not_local")
|
||||
_put_app_setting(GLINER_URL_KEY, chosen)
|
||||
if passage_profile_id is not None:
|
||||
token = (passage_profile_id or "").strip()
|
||||
if token:
|
||||
from provider_settings import get_profile
|
||||
|
||||
if not get_profile(token):
|
||||
raise ValueError("passage_profile_missing")
|
||||
_put_app_setting(PASSAGE_PROFILE_KEY, token)
|
||||
return get_local_ner_settings()
|
||||
|
||||
|
||||
def get_sense(profile_id: str, label: str) -> dict:
|
||||
key = normalize_label(label) or (label or "").strip()
|
||||
if not key:
|
||||
|
|
@ -376,21 +494,16 @@ def decision_for_mention(
|
|||
end: int | None,
|
||||
rendered: str = "",
|
||||
) -> str:
|
||||
"""identity / not_identity from prior review, or ask once for a new cue."""
|
||||
"""identity / not_identity from a stored cue, otherwise ask.
|
||||
|
||||
A sense with only identity hits must not lock a new cue (Sushi day 21).
|
||||
"""
|
||||
if not label:
|
||||
return _DECISION_ASK
|
||||
cue = cue_for_mention(user_body, start, end, label, rendered)
|
||||
learned = get_cue_decision(profile_id, label, cue)
|
||||
if learned:
|
||||
return learned
|
||||
sense = get_sense(profile_id, label)
|
||||
identity_shaped = cue in KINSHIP
|
||||
if sense["ambiguous"]:
|
||||
return _DECISION_ASK
|
||||
if sense["identity_hits"] > 0 and not sense["non_identity_hits"]:
|
||||
return _DECISION_IDENTITY
|
||||
if sense["non_identity_hits"] > 0 and not sense["identity_hits"]:
|
||||
return _DECISION_ASK if identity_shaped else _DECISION_NOT
|
||||
return _DECISION_ASK
|
||||
|
||||
|
||||
|
|
@ -533,6 +646,193 @@ def supplement_kinship_candidates(
|
|||
return extra_candidates, extra_mappings
|
||||
|
||||
|
||||
def merge_gliner_candidates(
|
||||
profile_id: str,
|
||||
candidates: list[dict],
|
||||
mappings: list[dict],
|
||||
user_body: str,
|
||||
rendered: str,
|
||||
spans: list[dict] | None,
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
"""Add GLiNER spans from the user line. Learned cues still win afterwards."""
|
||||
from gliner_client import gliner_identity_type, is_gliner_non_identity
|
||||
|
||||
extra_candidates = list(candidates)
|
||||
extra_mappings = list(mappings)
|
||||
if not spans:
|
||||
return extra_candidates, extra_mappings
|
||||
occupied: set[tuple[int, int, str]] = set()
|
||||
for item in extra_candidates:
|
||||
span = _resolve_user_span(
|
||||
user_body,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if span:
|
||||
occupied.add((span[0], span[1], (item.get("text") or "").casefold()))
|
||||
food_keys: set[tuple[int, int, str]] = set()
|
||||
for raw in spans:
|
||||
label = raw.get("text") or ""
|
||||
kind = raw.get("label") or ""
|
||||
try:
|
||||
start, end = int(raw["start"]), int(raw["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if start < 0 or end > len(user_body) or end <= start:
|
||||
continue
|
||||
if user_body[start:end].casefold() != label.casefold():
|
||||
resolved = _resolve_user_span(user_body, start, end, label, rendered)
|
||||
if not resolved:
|
||||
continue
|
||||
start, end = resolved
|
||||
label = user_body[start:end]
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
key = (start, end, label.casefold())
|
||||
if is_gliner_non_identity(kind):
|
||||
food_keys.add(key)
|
||||
continue
|
||||
entity_type = gliner_identity_type(kind)
|
||||
if not entity_type or key in occupied:
|
||||
continue
|
||||
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
|
||||
continue
|
||||
occupied.add(key)
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
extra_candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": entity_type,
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, start, end, label, rendered),
|
||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||
"suggested": _DECISION_IDENTITY,
|
||||
}
|
||||
)
|
||||
extra_mappings.append(
|
||||
{
|
||||
"token": _token_for_label(extra_mappings, label, entity_type),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": entity_type,
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
)
|
||||
for item in extra_candidates:
|
||||
span = _resolve_user_span(
|
||||
user_body,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if not span:
|
||||
continue
|
||||
key = (span[0], span[1], (item.get("text") or "").casefold())
|
||||
if key in food_keys and not attachment_kinship(user_body, span[0], span[1]):
|
||||
item["suggested"] = _DECISION_NOT
|
||||
for start, end, folded in food_keys:
|
||||
if (start, end, folded) in occupied:
|
||||
continue
|
||||
label = user_body[start:end]
|
||||
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
|
||||
continue
|
||||
occupied.add((start, end, folded))
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
extra_candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": "PERSON",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, start, end, label, rendered),
|
||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||
"suggested": _DECISION_NOT,
|
||||
}
|
||||
)
|
||||
extra_mappings.append(
|
||||
{
|
||||
"token": _token_for_label(extra_mappings, label),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": "PERSON",
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
)
|
||||
return extra_candidates, extra_mappings
|
||||
|
||||
|
||||
def apply_gliner_if_enabled(
|
||||
profile_id: str,
|
||||
candidates: list[dict],
|
||||
mappings: list[dict],
|
||||
user_body: str,
|
||||
rendered: str = "",
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
settings = get_local_ner_settings()
|
||||
if not settings.get("enabled"):
|
||||
return candidates, mappings
|
||||
from gliner_client import extract_entities
|
||||
|
||||
spans = extract_entities(user_body, url=settings.get("url"))
|
||||
if spans is None:
|
||||
return candidates, mappings
|
||||
return merge_gliner_candidates(profile_id, candidates, mappings, user_body, rendered, spans)
|
||||
|
||||
|
||||
def passage_provider():
|
||||
"""LLM profile for mini-passage decisions. Independent of the detect role."""
|
||||
import os
|
||||
|
||||
from providers import ProviderConfig, is_local_url
|
||||
from provider_settings import get_profile
|
||||
|
||||
profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
|
||||
if not profile_id:
|
||||
return None
|
||||
profile = get_profile(profile_id)
|
||||
if not profile:
|
||||
return None
|
||||
url = profile.get("url") or ""
|
||||
model = profile.get("model") or ""
|
||||
if not url or not model:
|
||||
return None
|
||||
local = is_local_url(url)
|
||||
key = ""
|
||||
if not local:
|
||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or os.environ.get("KANSHO_DETECT_PROVIDER_KEY") or "").strip()
|
||||
if not key:
|
||||
return None
|
||||
if not profile.get("zdr") or not profile.get("no_train"):
|
||||
return None
|
||||
return ProviderConfig(
|
||||
role="passage",
|
||||
name=profile.get("name") or ("ollama" if local else "openrouter"),
|
||||
mode="http",
|
||||
url=url,
|
||||
model=model,
|
||||
key=key,
|
||||
local=local,
|
||||
zdr=True if local else bool(profile.get("zdr")),
|
||||
no_train=True if local else bool(profile.get("no_train")),
|
||||
)
|
||||
|
||||
|
||||
def build_candidates(profile_id: str, mappings: list[dict], user_body: str, rendered: str = "") -> list[dict]:
|
||||
seen: set[tuple[int | None, int | None, str]] = set()
|
||||
items: list[dict] = []
|
||||
|
|
@ -721,6 +1021,8 @@ def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | No
|
|||
return {
|
||||
"id": row["id"],
|
||||
"candidates": payload.get("candidates") or [],
|
||||
"kind": payload.get("kind") or "diff",
|
||||
"purpose": payload.get("purpose") or "dialogue_turn",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -756,11 +1058,11 @@ def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: li
|
|||
|
||||
|
||||
def try_local_passage_decision(excerpt: str, label: str, examples: list[dict] | None = None) -> str | None:
|
||||
"""Return identity, not_identity, or None if no local model or unusable answer."""
|
||||
from providers import ProviderError, complete_chat, detect_provider
|
||||
"""Return identity, not_identity, or None. Uses the admin passage profile, not Detect."""
|
||||
from providers import ProviderError, complete_chat
|
||||
|
||||
config = detect_provider()
|
||||
if not config or not config.local or config.mode != "http":
|
||||
config = passage_provider()
|
||||
if not config or config.mode != "http":
|
||||
return None
|
||||
few_shot = ""
|
||||
lines = []
|
||||
|
|
|
|||
618
backend/detect_pipeline.py
Normal file
618
backend/detect_pipeline.py
Normal file
|
|
@ -0,0 +1,618 @@
|
|||
"""Student-first detect pipeline with optional teacher comparison.
|
||||
|
||||
Cues stay local. OpenRouter/GLiNER never create identities. Compact counters
|
||||
carry no plaintext labels.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from detect_learning import (
|
||||
MODE_LEARNING_MANUAL,
|
||||
MODE_STUDENT,
|
||||
_CUE_BARE,
|
||||
_DECISION_ASK,
|
||||
_DECISION_IDENTITY,
|
||||
_DECISION_NOT,
|
||||
_align_user_span,
|
||||
_mention_fields,
|
||||
_resolve_user_span,
|
||||
_token_for_label,
|
||||
_word_matches,
|
||||
apply_learned_decisions,
|
||||
auto_resolve_ambiguous,
|
||||
cue_for_mention,
|
||||
decision_for_mention,
|
||||
get_cue_decision,
|
||||
get_detect_operating_mode,
|
||||
get_local_ner_settings,
|
||||
get_sense,
|
||||
is_bypass_mode,
|
||||
is_learning_mode,
|
||||
learning_stage,
|
||||
merge_gliner_candidates,
|
||||
record_cue,
|
||||
supplement_kinship_candidates,
|
||||
)
|
||||
from entity_detect import DetectError, detect_personal_egress
|
||||
from identity_store import ENTITY_TYPES, is_maskable_label
|
||||
|
||||
KIND_DIFF = "diff"
|
||||
KIND_MANUAL = "manual"
|
||||
KIND_CONFIRM_REMOTE = "confirm_remote"
|
||||
|
||||
PROTECTING_TYPES = set(ENTITY_TYPES)
|
||||
FOOD_TYPES = frozenset(
|
||||
{"FOOD", "DISH", "MEAL", "CUISINE", "FOODSTUFF", "INGREDIENT", "food", "dish"}
|
||||
)
|
||||
TYPE_MAP = {
|
||||
"person": "PERSON",
|
||||
"place": "PLACE",
|
||||
"org": "ORG",
|
||||
"organization": "ORG",
|
||||
"project": "PROJECT",
|
||||
"food": "FOOD",
|
||||
"dish": "DISH",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MentionSpan:
|
||||
text: str
|
||||
start: int
|
||||
end: int
|
||||
entity_type: str
|
||||
protection: str
|
||||
source: str
|
||||
confirmed_cue: bool = False
|
||||
cue: str = _CUE_BARE
|
||||
|
||||
def key(self) -> tuple[int, int, str]:
|
||||
return (self.start, self.end, self.text.casefold())
|
||||
|
||||
|
||||
@dataclass
|
||||
class StudentResult:
|
||||
mappings: list[dict] = field(default_factory=list)
|
||||
candidates: list[dict] = field(default_factory=list)
|
||||
spans: list[MentionSpan] = field(default_factory=list)
|
||||
gliner_missing: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class TeacherResult:
|
||||
spans: list[MentionSpan] = field(default_factory=list)
|
||||
mappings: list[dict] = field(default_factory=list)
|
||||
ok: bool = True
|
||||
error: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineResult:
|
||||
mappings: list[dict]
|
||||
candidates: list[dict]
|
||||
kind: str = KIND_DIFF
|
||||
need_review: bool = False
|
||||
need_remote_confirm: bool = False
|
||||
counters: dict[str, int] = field(default_factory=dict)
|
||||
mode: str = ""
|
||||
|
||||
|
||||
def empty_counters() -> dict[str, int]:
|
||||
return {
|
||||
"diff_agree": 0,
|
||||
"diff_only_student": 0,
|
||||
"diff_only_teacher": 0,
|
||||
"diff_type_conflict": 0,
|
||||
"diff_new_identity": 0,
|
||||
"diff_cue_reinforced": 0,
|
||||
}
|
||||
|
||||
|
||||
def map_entity_type(raw: str) -> str:
|
||||
token = (raw or "").strip()
|
||||
if token.upper() in PROTECTING_TYPES or token.upper() in {item.upper() for item in FOOD_TYPES}:
|
||||
mapped = TYPE_MAP.get(token.lower())
|
||||
return mapped or token.upper()
|
||||
return TYPE_MAP.get(token.lower()) or token.upper()
|
||||
|
||||
|
||||
def protects(entity_type: str) -> bool:
|
||||
kind = map_entity_type(entity_type)
|
||||
if kind in FOOD_TYPES or kind.upper() in {item.upper() for item in FOOD_TYPES}:
|
||||
return False
|
||||
return kind in PROTECTING_TYPES
|
||||
|
||||
|
||||
def types_equivalent(left: str, right: str) -> bool:
|
||||
a, b = map_entity_type(left), map_entity_type(right)
|
||||
if a in FOOD_TYPES or b in FOOD_TYPES or a in {"FOOD", "DISH"} or b in {"FOOD", "DISH"}:
|
||||
return a == b
|
||||
if protects(a) and protects(b):
|
||||
return True
|
||||
return a == b
|
||||
|
||||
|
||||
def _contains(outer: MentionSpan, inner: MentionSpan) -> bool:
|
||||
return outer.start <= inner.start and inner.end <= outer.end and outer.end > outer.start
|
||||
|
||||
|
||||
def same_mention(left: MentionSpan, right: MentionSpan) -> bool:
|
||||
if left.text.casefold() == right.text.casefold() and left.start == right.start and left.end == right.end:
|
||||
return True
|
||||
if _contains(left, right) or _contains(right, left):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _collapse_same_protection(spans: list[MentionSpan]) -> list[MentionSpan]:
|
||||
if not spans:
|
||||
return []
|
||||
remaining = list(spans)
|
||||
chosen: list[MentionSpan] = []
|
||||
while remaining:
|
||||
current = remaining.pop(0)
|
||||
cluster = [current]
|
||||
rest = []
|
||||
for item in remaining:
|
||||
if same_mention(current, item):
|
||||
cluster.append(item)
|
||||
else:
|
||||
rest.append(item)
|
||||
remaining = rest
|
||||
protections = {item.protection for item in cluster}
|
||||
if len(protections) > 1:
|
||||
chosen.extend(cluster)
|
||||
continue
|
||||
chosen.append(max(cluster, key=lambda item: (item.end - item.start, -item.start)))
|
||||
return chosen
|
||||
|
||||
|
||||
def _user_span_for_mapping(mapping: dict, user_body: str, rendered: str = "") -> tuple[int, int] | None:
|
||||
return _resolve_user_span(
|
||||
user_body,
|
||||
mapping.get("start"),
|
||||
mapping.get("end"),
|
||||
mapping.get("local_label") or "",
|
||||
rendered,
|
||||
)
|
||||
|
||||
|
||||
def _mapping_from_user_span(
|
||||
mappings: list[dict],
|
||||
label: str,
|
||||
user_body: str,
|
||||
start: int,
|
||||
end: int,
|
||||
entity_type: str,
|
||||
rendered: str,
|
||||
) -> dict:
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
kind = entity_type if entity_type in ENTITY_TYPES else "PERSON"
|
||||
return {
|
||||
"token": _token_for_label(mappings, label, kind),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": kind,
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
|
||||
|
||||
def run_student_path(profile_id: str, user_body: str, rendered: str = "") -> StudentResult:
|
||||
mappings: list[dict] = []
|
||||
candidates: list[dict] = []
|
||||
spans: list[MentionSpan] = []
|
||||
text = user_body or ""
|
||||
for match in _word_matches(text):
|
||||
label = match.group(0)
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
start, end = match.start(), match.end()
|
||||
verdict = decision_for_mention(profile_id, label, text, start, end, rendered)
|
||||
cue = cue_for_mention(text, start, end, label, rendered)
|
||||
if verdict == _DECISION_IDENTITY:
|
||||
mappings.append(_mapping_from_user_span(mappings, label, text, start, end, "PERSON", rendered))
|
||||
spans.append(
|
||||
MentionSpan(label, start, end, "PERSON", _DECISION_IDENTITY, "cue", True, cue)
|
||||
)
|
||||
elif verdict == _DECISION_NOT:
|
||||
spans.append(
|
||||
MentionSpan(label, start, end, "FOOD", _DECISION_NOT, "cue", True, cue)
|
||||
)
|
||||
elif get_sense(profile_id, label)["ambiguous"]:
|
||||
aligned = _align_user_span(rendered, text, start, end)
|
||||
candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": "PERSON",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(text, start, end, label, rendered),
|
||||
"ambiguous": True,
|
||||
}
|
||||
)
|
||||
candidates, mappings = supplement_kinship_candidates(
|
||||
profile_id, candidates, mappings, text, rendered
|
||||
)
|
||||
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
||||
bound = {(span.start, span.end, span.text.casefold()) for span in spans}
|
||||
for item in candidates:
|
||||
resolved = _resolve_user_span(
|
||||
text,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if resolved:
|
||||
bound.add((resolved[0], resolved[1], (item.get("text") or "").casefold()))
|
||||
settings = get_local_ner_settings()
|
||||
gliner_missing = False
|
||||
if settings.get("enabled"):
|
||||
from gliner_client import extract_entities
|
||||
|
||||
raw = extract_entities(text, url=settings.get("url"))
|
||||
if raw is None:
|
||||
gliner_missing = True
|
||||
else:
|
||||
usable = []
|
||||
for item in raw:
|
||||
try:
|
||||
start, end = int(item["start"]), int(item["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
label = (item.get("text") or "").casefold()
|
||||
if any(
|
||||
not (end <= left or start >= right) and folded == label
|
||||
for left, right, folded in bound
|
||||
) or any(left <= start and end <= right for left, right, _folded in bound):
|
||||
continue
|
||||
usable.append(item)
|
||||
candidates, mappings = merge_gliner_candidates(
|
||||
profile_id, candidates, mappings, text, rendered, usable
|
||||
)
|
||||
candidates, mappings = apply_learned_decisions(profile_id, candidates, mappings, text, rendered)
|
||||
for item in candidates:
|
||||
resolved = _resolve_user_span(
|
||||
text,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if not resolved:
|
||||
continue
|
||||
suggested = item.get("suggested") or _DECISION_IDENTITY
|
||||
kind = item.get("entity_type") or "PERSON"
|
||||
protection = _DECISION_NOT if suggested == _DECISION_NOT else _DECISION_IDENTITY
|
||||
entity = "FOOD" if protection == _DECISION_NOT else kind
|
||||
spans.append(
|
||||
MentionSpan(
|
||||
item.get("text") or "",
|
||||
resolved[0],
|
||||
resolved[1],
|
||||
entity,
|
||||
protection,
|
||||
"candidate",
|
||||
False,
|
||||
item.get("cue") or _CUE_BARE,
|
||||
)
|
||||
)
|
||||
return StudentResult(
|
||||
mappings=mappings,
|
||||
candidates=candidates,
|
||||
spans=_collapse_same_protection(spans),
|
||||
gliner_missing=gliner_missing,
|
||||
)
|
||||
|
||||
|
||||
def blank_cue_safe_forms(profile_id: str, user_body: str, rendered: str = "") -> str:
|
||||
chars = list(user_body or "")
|
||||
for match in _word_matches(user_body or ""):
|
||||
label = match.group(0)
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
verdict = decision_for_mention(
|
||||
profile_id, label, user_body, match.start(), match.end(), rendered
|
||||
)
|
||||
if verdict not in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||
continue
|
||||
for index in range(match.start(), match.end()):
|
||||
chars[index] = " "
|
||||
return "".join(chars)
|
||||
|
||||
|
||||
def _teacher_span(label: str, start: int, end: int, entity_type: str) -> MentionSpan:
|
||||
kind = map_entity_type(entity_type)
|
||||
protection = _DECISION_IDENTITY if protects(kind) else _DECISION_NOT
|
||||
return MentionSpan(label, start, end, kind, protection, "teacher", False)
|
||||
|
||||
|
||||
def run_teacher(
|
||||
profile_id: str,
|
||||
user_body: str,
|
||||
*,
|
||||
stage: int = 1,
|
||||
rendered: str = "",
|
||||
) -> TeacherResult:
|
||||
source = user_body or ""
|
||||
if stage >= 2:
|
||||
source = blank_cue_safe_forms(profile_id, user_body, rendered)
|
||||
try:
|
||||
outcome = detect_personal_egress(profile_id, source)
|
||||
except DetectError as exc:
|
||||
return TeacherResult(ok=False, error=exc.code)
|
||||
spans: list[MentionSpan] = []
|
||||
for mapping in outcome.mappings or []:
|
||||
label = mapping.get("local_label") or ""
|
||||
start, end = mapping.get("start"), mapping.get("end")
|
||||
resolved = None
|
||||
if start is not None and end is not None:
|
||||
left, right = int(start), int(end)
|
||||
if 0 <= left < right <= len(user_body) and user_body[left:right].casefold() == label.casefold():
|
||||
resolved = (left, right)
|
||||
if not resolved:
|
||||
resolved = _resolve_user_span(user_body, start, end, label, user_body)
|
||||
if not resolved:
|
||||
continue
|
||||
spans.append(
|
||||
_teacher_span(label, resolved[0], resolved[1], mapping.get("entity_type") or "PERSON")
|
||||
)
|
||||
omitted = list((outcome.stats.omitted_non_identity_types or []) if outcome.stats else [])
|
||||
for kind in omitted:
|
||||
if map_entity_type(kind) in FOOD_TYPES or kind.upper() in {"FOOD", "DISH"}:
|
||||
continue
|
||||
return TeacherResult(spans=_collapse_same_protection(spans), mappings=list(outcome.mappings or []), ok=True)
|
||||
|
||||
|
||||
def _candidate_from_span(
|
||||
span: MentionSpan,
|
||||
user_body: str,
|
||||
rendered: str,
|
||||
*,
|
||||
suggested: str,
|
||||
reason: str,
|
||||
) -> dict:
|
||||
aligned = _align_user_span(rendered, user_body, span.start, span.end)
|
||||
kind = span.entity_type if span.entity_type in ENTITY_TYPES else "PERSON"
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": span.text,
|
||||
"entity_type": kind,
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, span.start, span.end, span.text, rendered),
|
||||
"ambiguous": False,
|
||||
"suggested": suggested,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
|
||||
def compare_mentions(
|
||||
profile_id: str,
|
||||
user_body: str,
|
||||
student_spans: list[MentionSpan],
|
||||
teacher_spans: list[MentionSpan],
|
||||
rendered: str = "",
|
||||
) -> tuple[list[dict], dict[str, int]]:
|
||||
counters = empty_counters()
|
||||
review: list[dict] = []
|
||||
used_teacher: set[int] = set()
|
||||
student_list = _collapse_same_protection(student_spans)
|
||||
teacher_list = _collapse_same_protection(teacher_spans)
|
||||
|
||||
def take_teacher(index: int) -> MentionSpan:
|
||||
used_teacher.add(index)
|
||||
return teacher_list[index]
|
||||
|
||||
for student in student_list:
|
||||
match_index = next(
|
||||
(
|
||||
index
|
||||
for index, teacher in enumerate(teacher_list)
|
||||
if index not in used_teacher and same_mention(student, teacher)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if match_index is None:
|
||||
counters["diff_only_student"] += 1
|
||||
if student.confirmed_cue:
|
||||
continue
|
||||
if student.protection == _DECISION_IDENTITY:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_student"
|
||||
)
|
||||
)
|
||||
continue
|
||||
teacher = take_teacher(match_index)
|
||||
if student.protection != teacher.protection or (
|
||||
protects(student.entity_type) != protects(teacher.entity_type)
|
||||
):
|
||||
counters["diff_type_conflict"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student,
|
||||
user_body,
|
||||
rendered,
|
||||
suggested=student.protection,
|
||||
reason="type_conflict",
|
||||
)
|
||||
)
|
||||
continue
|
||||
if types_equivalent(student.entity_type, teacher.entity_type) or (
|
||||
student.protection == teacher.protection
|
||||
):
|
||||
counters["diff_agree"] += 1
|
||||
if student.confirmed_cue:
|
||||
record_cue(profile_id, student.text, student.cue, student.protection)
|
||||
counters["diff_cue_reinforced"] += 1
|
||||
elif student.protection == _DECISION_IDENTITY:
|
||||
learned = get_cue_decision(profile_id, student.text, student.cue)
|
||||
if learned == _DECISION_IDENTITY:
|
||||
record_cue(profile_id, student.text, student.cue, _DECISION_IDENTITY)
|
||||
counters["diff_cue_reinforced"] += 1
|
||||
else:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="new_identity"
|
||||
)
|
||||
)
|
||||
for index, teacher in enumerate(teacher_list):
|
||||
if index in used_teacher:
|
||||
continue
|
||||
counters["diff_only_teacher"] += 1
|
||||
cue = cue_for_mention(user_body, teacher.start, teacher.end, teacher.text, rendered)
|
||||
learned = get_cue_decision(profile_id, teacher.text, cue)
|
||||
if learned:
|
||||
continue
|
||||
if teacher.protection == _DECISION_IDENTITY:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
teacher, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_teacher"
|
||||
)
|
||||
)
|
||||
return review, counters
|
||||
|
||||
|
||||
def _remote_confirm_candidate(user_body: str) -> dict:
|
||||
snippet = (user_body or "")[:80]
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": "Klartext-Detect",
|
||||
"entity_type": "PERSON",
|
||||
"start": 0,
|
||||
"end": 0,
|
||||
"excerpt": snippet,
|
||||
"highlight_start": None,
|
||||
"highlight_end": None,
|
||||
"user_start": None,
|
||||
"user_end": None,
|
||||
"cue": _CUE_BARE,
|
||||
"ambiguous": False,
|
||||
"suggested": _DECISION_NOT,
|
||||
"reason": KIND_CONFIRM_REMOTE,
|
||||
}
|
||||
|
||||
|
||||
def prepare_detect_pipeline(profile_id: str, user_body: str, rendered: str = "") -> PipelineResult:
|
||||
mode = get_detect_operating_mode()
|
||||
counters = empty_counters()
|
||||
if is_bypass_mode(mode):
|
||||
return PipelineResult(mappings=[], candidates=[], kind=KIND_DIFF, mode=mode, counters=counters)
|
||||
student = run_student_path(profile_id, user_body, rendered)
|
||||
if mode == MODE_STUDENT:
|
||||
if student.gliner_missing:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=[_remote_confirm_candidate(user_body)],
|
||||
kind=KIND_CONFIRM_REMOTE,
|
||||
need_review=True,
|
||||
need_remote_confirm=True,
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=[],
|
||||
kind=KIND_DIFF,
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
if mode == MODE_LEARNING_MANUAL or student.gliner_missing:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=student.candidates,
|
||||
kind=KIND_MANUAL,
|
||||
need_review=bool(student.candidates),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
teacher = run_teacher(profile_id, user_body, stage=learning_stage(mode), rendered=rendered)
|
||||
if not teacher.ok:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=student.candidates,
|
||||
kind=KIND_MANUAL,
|
||||
need_review=bool(student.candidates),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
review, counters = compare_mentions(
|
||||
profile_id, user_body, student.spans, teacher.spans, rendered
|
||||
)
|
||||
merged = list(student.mappings)
|
||||
seen = {
|
||||
(
|
||||
item.get("start"),
|
||||
item.get("end"),
|
||||
(item.get("local_label") or "").casefold(),
|
||||
)
|
||||
for item in merged
|
||||
}
|
||||
for mapping in teacher.mappings:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in seen:
|
||||
continue
|
||||
label = mapping.get("local_label") or ""
|
||||
resolved = _resolve_user_span(
|
||||
user_body, mapping.get("start"), mapping.get("end"), label, user_body
|
||||
)
|
||||
if resolved:
|
||||
cue = cue_for_mention(user_body, resolved[0], resolved[1], label, rendered)
|
||||
if get_cue_decision(profile_id, label, cue) == _DECISION_NOT:
|
||||
continue
|
||||
merged.append(mapping)
|
||||
seen.add(key)
|
||||
return PipelineResult(
|
||||
mappings=merged,
|
||||
candidates=review,
|
||||
kind=KIND_DIFF,
|
||||
need_review=bool(review),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
|
||||
def compact_diff_counters(counters: dict[str, int] | None) -> dict[str, int]:
|
||||
payload = empty_counters()
|
||||
for key in payload:
|
||||
payload[key] = int((counters or {}).get(key) or 0)
|
||||
return payload
|
||||
|
||||
|
||||
def apply_remote_confirm(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
|
||||
"""After the user allows plaintext detect, run teacher on the user line."""
|
||||
approved = any((item.get("decision") or "").strip() == _DECISION_IDENTITY for item in decisions)
|
||||
if not approved:
|
||||
raise DetectError(
|
||||
"remote_detect_declined",
|
||||
"Klartext-Detect wurde nicht bestätigt.",
|
||||
403,
|
||||
)
|
||||
user_body = pending.get("user_body") or ""
|
||||
teacher = run_teacher(profile_id, user_body, stage=1)
|
||||
if not teacher.ok:
|
||||
raise DetectError(teacher.error or "detect_provider_unavailable", "Lehrer-Detect ist nicht erreichbar.")
|
||||
merged = list(pending.get("mappings") or [])
|
||||
seen = {
|
||||
(item.get("start"), item.get("end"), (item.get("local_label") or "").casefold())
|
||||
for item in merged
|
||||
}
|
||||
for mapping in teacher.mappings:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in seen:
|
||||
continue
|
||||
merged.append(mapping)
|
||||
seen.add(key)
|
||||
return merged
|
||||
|
|
@ -8,22 +8,18 @@ from context_builder import assemble_text, build_internal_context, is_closing_tu
|
|||
from conversation_signals import infer_signals
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from detect_learning import (
|
||||
MODE_LEARNING,
|
||||
apply_learned_decisions,
|
||||
apply_review_decisions,
|
||||
auto_resolve_ambiguous,
|
||||
build_candidates,
|
||||
confirm_known_identity_spans,
|
||||
drop_pending,
|
||||
get_detect_operating_mode,
|
||||
is_bypass_mode,
|
||||
load_pending,
|
||||
save_pending,
|
||||
supplement_kinship_candidates,
|
||||
suppress_known_non_identity,
|
||||
)
|
||||
from detect_pipeline import KIND_CONFIRM_REMOTE, prepare_detect_pipeline
|
||||
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError, detect_personal_egress
|
||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError
|
||||
from writing_profile_store import remember_dialogue_style
|
||||
from profile_review import consider_dialogue
|
||||
|
||||
|
|
@ -331,6 +327,7 @@ def _finish_turn(
|
|||
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||
call_traces: list[dict] = []
|
||||
working = dict(assembled)
|
||||
working["user_body"] = user.get("body") or working.get("user_body") or ""
|
||||
try:
|
||||
while calls < 2:
|
||||
result = execute_prompt(
|
||||
|
|
@ -431,24 +428,22 @@ def _finish_turn(
|
|||
}
|
||||
|
||||
|
||||
def _review_pause_payload(review_id: str, candidates: list[dict], kind: str, purpose: str = "dialogue_turn") -> dict:
|
||||
return {
|
||||
"id": review_id,
|
||||
"candidates": candidates,
|
||||
"kind": kind,
|
||||
"purpose": purpose,
|
||||
}
|
||||
|
||||
|
||||
def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled: dict, prompt: dict) -> dict | None:
|
||||
preview = preview_prompt(prompt, assembled)
|
||||
rendered = preview.get("rendered") or ""
|
||||
try:
|
||||
outcome = detect_personal_egress(profile_id, rendered)
|
||||
except DetectError:
|
||||
return None
|
||||
user_body = user.get("body") or ""
|
||||
mappings = list(outcome.mappings or [])
|
||||
candidates = build_candidates(profile_id, mappings, user_body, rendered)
|
||||
candidates, mappings = supplement_kinship_candidates(
|
||||
profile_id, candidates, mappings, user_body, rendered
|
||||
)
|
||||
candidates, mappings = apply_learned_decisions(
|
||||
profile_id, candidates, mappings, user_body, rendered
|
||||
)
|
||||
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
||||
if not candidates:
|
||||
result = prepare_detect_pipeline(profile_id, user_body, rendered)
|
||||
mappings = list(result.mappings or [])
|
||||
if not result.need_review and not result.candidates:
|
||||
confirm_known_identity_spans(profile_id, mappings, user_body)
|
||||
mappings = suppress_known_non_identity(profile_id, mappings, user_body)
|
||||
return _finish_turn(
|
||||
|
|
@ -465,8 +460,11 @@ def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled
|
|||
user.get("id") or "",
|
||||
{
|
||||
"mappings": mappings,
|
||||
"candidates": candidates,
|
||||
"candidates": result.candidates,
|
||||
"user_body": user_body,
|
||||
"kind": result.kind,
|
||||
"counters": result.counters,
|
||||
"purpose": "dialogue_turn",
|
||||
},
|
||||
)
|
||||
return {
|
||||
|
|
@ -475,10 +473,7 @@ def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled
|
|||
"assistant": None,
|
||||
"calls": 0,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
"pending_mask_review": {
|
||||
"id": review_id,
|
||||
"candidates": candidates,
|
||||
},
|
||||
"pending_mask_review": _review_pause_payload(review_id, result.candidates, result.kind),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -494,10 +489,11 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
|||
)
|
||||
assembled = assemble_text(context)
|
||||
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||
if get_detect_operating_mode() == MODE_LEARNING:
|
||||
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
|
||||
if paused is not None:
|
||||
return paused
|
||||
if is_bypass_mode():
|
||||
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
|
||||
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
|
||||
if paused is not None:
|
||||
return paused
|
||||
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
|
||||
|
||||
|
||||
|
|
@ -505,7 +501,17 @@ def continue_turn(profile_id: str, conversation_id: str, review_id: str, decisio
|
|||
pending = load_pending(profile_id, review_id)
|
||||
if not pending or pending.get("conversation_id") != conversation_id:
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
if pending.get("purpose") == "journal_generate":
|
||||
raise EngineError("mask_review_missing", "Diese Prüfung gehört zum Journalentwurf.", 409)
|
||||
if pending.get("kind") == KIND_CONFIRM_REMOTE:
|
||||
try:
|
||||
from detect_pipeline import apply_remote_confirm
|
||||
|
||||
mappings = apply_remote_confirm(profile_id, pending, decisions)
|
||||
except DetectError as exc:
|
||||
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
|
||||
else:
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
drop_pending(profile_id, review_id)
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
context = build_internal_context(
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ def execute_prompt(
|
|||
budget=None,
|
||||
diagnostics: dict[str, Any] | None = None,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
detect_counters: dict | None = None,
|
||||
) -> dict:
|
||||
preview = preview_prompt(prompt, context)
|
||||
feature_id = prompt.get("required_feature") or "ai_calls"
|
||||
|
|
@ -101,6 +102,7 @@ def execute_prompt(
|
|||
payload={
|
||||
"rendered": preview["rendered"],
|
||||
"source_text": source_text,
|
||||
"user_body": (ctx.get("user_body") or user_source_text(ctx.get("dialogue_context") or "")).strip(),
|
||||
"privacy_tokens": preview["privacy_tokens"],
|
||||
"prompt_slug": prompt.get("slug"),
|
||||
"prompt_revision": prompt.get("seed_revision") or "",
|
||||
|
|
@ -109,6 +111,7 @@ def execute_prompt(
|
|||
"budget": budget,
|
||||
"diagnostics": diagnostics or {},
|
||||
"precomputed_mappings": precomputed_mappings,
|
||||
"detect_counters": detect_counters or {},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
|
|
|||
121
backend/gliner_client.py
Normal file
121
backend/gliner_client.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""Stateless client for the LAN GLiNER spike. Fail-soft; never logs request text."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
import httpx
|
||||
|
||||
from providers import is_local_url
|
||||
|
||||
log = logging.getLogger("gliner_client")
|
||||
|
||||
DEFAULT_URL = "http://192.168.2.144:8090"
|
||||
DEFAULT_LABELS = ("person", "place", "organization", "project", "dish", "food")
|
||||
IDENTITY_LABELS = {
|
||||
"person": "PERSON",
|
||||
"place": "PLACE",
|
||||
"organization": "ORG",
|
||||
"project": "PROJECT",
|
||||
}
|
||||
NON_IDENTITY_LABELS = frozenset({"dish", "food"})
|
||||
EXTRACT_TIMEOUT = 8.0
|
||||
HEALTH_TIMEOUT = 2.0
|
||||
|
||||
|
||||
def normalize_base_url(url: str | None) -> str:
|
||||
raw = (url or DEFAULT_URL).strip() or DEFAULT_URL
|
||||
parsed = urlparse(raw)
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
return DEFAULT_URL
|
||||
path = (parsed.path or "").rstrip("/")
|
||||
return urlunparse((parsed.scheme, parsed.netloc, path, "", "", ""))
|
||||
|
||||
|
||||
def gliner_identity_type(label: str) -> str | None:
|
||||
return IDENTITY_LABELS.get((label or "").strip().lower())
|
||||
|
||||
|
||||
def is_gliner_non_identity(label: str) -> bool:
|
||||
return (label or "").strip().lower() in NON_IDENTITY_LABELS
|
||||
|
||||
|
||||
def health(url: str | None = None) -> dict[str, Any]:
|
||||
"""Ping GLiNER. Never include request text. Fail-soft."""
|
||||
base = normalize_base_url(url)
|
||||
if not is_local_url(base):
|
||||
return {"ok": False, "status": "rejected", "reason": "not_local"}
|
||||
try:
|
||||
response = httpx.get(f"{base}/health", timeout=HEALTH_TIMEOUT)
|
||||
except httpx.HTTPError:
|
||||
return {"ok": False, "status": "unreachable", "url": base}
|
||||
if response.status_code >= 400:
|
||||
return {"ok": False, "status": "http_error", "http_status": response.status_code}
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return {"ok": False, "status": "invalid"}
|
||||
if not isinstance(payload, dict):
|
||||
return {"ok": False, "status": "invalid"}
|
||||
status = (payload.get("status") or "").strip() or "ok"
|
||||
return {
|
||||
"ok": status == "ok",
|
||||
"status": status,
|
||||
"model": payload.get("model") or "",
|
||||
"offline": bool(payload.get("offline")),
|
||||
}
|
||||
|
||||
|
||||
def extract_entities(text: str, url: str | None = None) -> list[dict] | None:
|
||||
"""Return spans or None if the service cannot be used. Empty list is a valid miss."""
|
||||
body = text or ""
|
||||
if not body.strip():
|
||||
return []
|
||||
base = normalize_base_url(url)
|
||||
if not is_local_url(base):
|
||||
log.info("gliner skipped: url is not local")
|
||||
return None
|
||||
try:
|
||||
response = httpx.post(
|
||||
f"{base}/v1/extract",
|
||||
json={"text": body, "labels": list(DEFAULT_LABELS)},
|
||||
timeout=EXTRACT_TIMEOUT,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
log.info("gliner extract unreachable chars=%s", len(body))
|
||||
return None
|
||||
if response.status_code >= 400:
|
||||
log.info("gliner extract http=%s chars=%s", response.status_code, len(body))
|
||||
return None
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return None
|
||||
raw = payload.get("spans") if isinstance(payload, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
spans = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
try:
|
||||
start = int(item["start"])
|
||||
end = int(item["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
label = str(item.get("text") or "")
|
||||
kind = str(item.get("label") or "")
|
||||
if start < 0 or end <= start or not label or not kind:
|
||||
continue
|
||||
spans.append(
|
||||
{
|
||||
"text": label,
|
||||
"start": start,
|
||||
"end": end,
|
||||
"label": kind,
|
||||
"score": item.get("score"),
|
||||
}
|
||||
)
|
||||
log.info("gliner extract chars=%s spans=%s", len(body), len(spans))
|
||||
return spans
|
||||
|
|
@ -5,8 +5,17 @@ import re
|
|||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from detect_learning import (
|
||||
apply_review_decisions,
|
||||
drop_pending,
|
||||
is_bypass_mode,
|
||||
load_pending,
|
||||
save_pending,
|
||||
)
|
||||
from detect_pipeline import KIND_CONFIRM_REMOTE, apply_remote_confirm, prepare_detect_pipeline
|
||||
from dialogue_store import StoreError, list_conversations_for_day, list_messages
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from entity_detect import DetectError
|
||||
from journal_policy import require_explicit_generate, source_conversation_ids
|
||||
from identity_store import is_maskable_label, list_mappings, mapping_spellings
|
||||
from journal_body import clean_title
|
||||
|
|
@ -372,6 +381,7 @@ def generate_draft(
|
|||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
) -> dict:
|
||||
try:
|
||||
draft = _generate_draft(
|
||||
|
|
@ -382,6 +392,7 @@ def generate_draft(
|
|||
explicit=explicit,
|
||||
generation_selection=generation_selection,
|
||||
remember_generation_selection=remember_generation_selection,
|
||||
precomputed_mappings=precomputed_mappings,
|
||||
)
|
||||
except EngineError as exc:
|
||||
persist_engine_error(
|
||||
|
|
@ -394,6 +405,8 @@ def generate_draft(
|
|||
extra={"conversation_ids": conversation_ids or []},
|
||||
)
|
||||
raise
|
||||
if draft.get("pending_mask_review"):
|
||||
return draft
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
|
|
@ -423,6 +436,7 @@ def _generate_draft(
|
|||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
) -> dict:
|
||||
require_explicit_generate(explicit)
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
|
|
@ -564,6 +578,44 @@ def _generate_draft(
|
|||
)
|
||||
rendered_preview = preview_prompt(narrate_prompt, assembled)
|
||||
assert_template_resolved(rendered_preview.get("rendered") or "")
|
||||
detect_mappings = precomputed_mappings
|
||||
if detect_mappings is None and not is_bypass_mode():
|
||||
user_text = "\n".join(source_user)
|
||||
pipe = prepare_detect_pipeline(profile_id, user_text, "")
|
||||
if (pipe.need_review or pipe.candidates) and selected:
|
||||
review_id = save_pending(
|
||||
profile_id,
|
||||
selected[0],
|
||||
"",
|
||||
{
|
||||
"mappings": pipe.mappings,
|
||||
"candidates": pipe.candidates,
|
||||
"user_body": user_text,
|
||||
"kind": pipe.kind,
|
||||
"counters": pipe.counters,
|
||||
"purpose": "journal_generate",
|
||||
"generate": {
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected,
|
||||
"include_existing": include_existing,
|
||||
"generation_selection": generation_selection,
|
||||
"remember_generation_selection": remember_generation_selection,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"pending_mask_review": {
|
||||
"id": review_id,
|
||||
"candidates": pipe.candidates,
|
||||
"kind": pipe.kind,
|
||||
"purpose": "journal_generate",
|
||||
},
|
||||
"run_log": run_log,
|
||||
"trace": None,
|
||||
}
|
||||
detect_mappings = pipe.mappings
|
||||
assembled = dict(assembled)
|
||||
assembled["user_body"] = "\n".join(source_user)
|
||||
narrate_result = execute_prompt(
|
||||
narrate_prompt,
|
||||
profile_id,
|
||||
|
|
@ -573,6 +625,7 @@ def _generate_draft(
|
|||
max_tokens=narrate_budget.reserved_output_tokens,
|
||||
disable_context_compression=True,
|
||||
budget=narrate_budget,
|
||||
precomputed_mappings=detect_mappings,
|
||||
)
|
||||
_stamp_log(run_log, "journal_generate", _log_items(narrate_result))
|
||||
title, body = _split_title(narrate_result.get("content") or "")
|
||||
|
|
@ -745,3 +798,30 @@ def _generate_draft(
|
|||
draft["run_log"] = run_log
|
||||
draft["trace"] = bundle
|
||||
return draft
|
||||
|
||||
|
||||
def continue_journal_generate(profile_id: str, journal_day_id: str, review_id: str, decisions: list[dict]) -> dict:
|
||||
pending = load_pending(profile_id, review_id)
|
||||
if not pending or pending.get("purpose") != "journal_generate":
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
generate = pending.get("generate") or {}
|
||||
if generate.get("journal_day_id") != journal_day_id:
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
if pending.get("kind") == KIND_CONFIRM_REMOTE:
|
||||
try:
|
||||
mappings = apply_remote_confirm(profile_id, pending, decisions)
|
||||
except DetectError as exc:
|
||||
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
|
||||
else:
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
drop_pending(profile_id, review_id)
|
||||
return generate_draft(
|
||||
profile_id,
|
||||
journal_day_id,
|
||||
conversation_ids=generate.get("conversation_ids"),
|
||||
include_existing=bool(generate.get("include_existing")),
|
||||
explicit=True,
|
||||
generation_selection=generate.get("generation_selection"),
|
||||
remember_generation_selection=bool(generate.get("remember_generation_selection")),
|
||||
precomputed_mappings=mappings,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -98,6 +98,12 @@ COMPACT_DIAGNOSTIC_KEYS = (
|
|||
"invalid_entity_type",
|
||||
"omitted_non_identity_types",
|
||||
"detect_cost_unknown",
|
||||
"diff_agree",
|
||||
"diff_only_student",
|
||||
"diff_only_teacher",
|
||||
"diff_type_conflict",
|
||||
"diff_new_identity",
|
||||
"diff_cue_reinforced",
|
||||
"generation_selection",
|
||||
"style_application",
|
||||
)
|
||||
|
|
@ -1059,10 +1065,12 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
|||
try:
|
||||
if precomputed is not None:
|
||||
mappings = list(precomputed)
|
||||
extra = dict((request.payload or {}).get("detect_counters") or {})
|
||||
detect_stats = {
|
||||
"detect_note": "precomputed_learning_review",
|
||||
"full_detection_coverage": True,
|
||||
"semantic_identity_guaranteed": False,
|
||||
**{key: extra[key] for key in extra if key.startswith("diff_")},
|
||||
}
|
||||
local_identities = [
|
||||
{
|
||||
|
|
@ -1075,12 +1083,37 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
|||
]
|
||||
detect_name = "learning_review"
|
||||
else:
|
||||
outcome = detect_personal_egress(request.profile_id, rendered)
|
||||
mappings = outcome.mappings
|
||||
detect_stats = outcome.stats.public()
|
||||
detect_name = outcome.stats.detect_provider
|
||||
detect_note = outcome.stats.detect_note
|
||||
local_identities = outcome.local_identities
|
||||
from detect_learning import is_bypass_mode
|
||||
from detect_pipeline import compact_diff_counters, prepare_detect_pipeline
|
||||
|
||||
user_body = str((request.payload or {}).get("user_body") or "").strip()
|
||||
if request.profile_id and not is_bypass_mode() and user_body:
|
||||
pipe = prepare_detect_pipeline(request.profile_id, user_body, rendered)
|
||||
mappings = list(pipe.mappings or [])
|
||||
detect_stats = {
|
||||
"detect_note": "student_path" if not pipe.need_review else "student_path_unreviewed",
|
||||
"full_detection_coverage": True,
|
||||
"semantic_identity_guaranteed": False,
|
||||
**compact_diff_counters(pipe.counters),
|
||||
}
|
||||
detect_name = "student_path"
|
||||
detect_note = detect_stats["detect_note"]
|
||||
local_identities = [
|
||||
{
|
||||
"local_label": item.get("local_label"),
|
||||
"token": item.get("token"),
|
||||
"entity_type": item.get("entity_type"),
|
||||
"source": item.get("source"),
|
||||
}
|
||||
for item in mappings
|
||||
]
|
||||
else:
|
||||
outcome = detect_personal_egress(request.profile_id, rendered)
|
||||
mappings = outcome.mappings
|
||||
detect_stats = outcome.stats.public()
|
||||
detect_name = outcome.stats.detect_provider
|
||||
detect_note = outcome.stats.detect_note
|
||||
local_identities = outcome.local_identities
|
||||
except DetectError as exc:
|
||||
detect_stats = dict(exc.diagnostics or {})
|
||||
_log_event(
|
||||
|
|
|
|||
|
|
@ -285,6 +285,10 @@ def delete_profile(profile_id: str) -> None:
|
|||
"profile_in_use",
|
||||
f"Profil ist noch der Stufe {roles} zugeordnet. Zuerst ein anderes Profil wählen.",
|
||||
)
|
||||
from detect_learning import get_local_ner_settings, set_local_ner_settings
|
||||
|
||||
if get_local_ner_settings().get("passage_profile_id") == profile_id:
|
||||
set_local_ner_settings(passage_profile_id="")
|
||||
conn.execute("DELETE FROM llm_profiles WHERE id = ?", (profile_id,))
|
||||
|
||||
|
||||
|
|
@ -462,4 +466,11 @@ def public_status() -> dict:
|
|||
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
|
||||
"ollama_url": OLLAMA_LAN_URL,
|
||||
"detect_operating_mode": get_detect_operating_mode(),
|
||||
"local_ner": _local_ner_status(),
|
||||
}
|
||||
|
||||
|
||||
def _local_ner_status() -> dict:
|
||||
from detect_learning import get_local_ner_settings
|
||||
|
||||
return get_local_ner_settings()
|
||||
|
|
|
|||
|
|
@ -231,6 +231,12 @@ class DetectModeWrite(BaseModel):
|
|||
mode: str
|
||||
|
||||
|
||||
class LocalNerWrite(BaseModel):
|
||||
enabled: bool | None = None
|
||||
url: str | None = None
|
||||
passage_profile_id: str | None = None
|
||||
|
||||
|
||||
@router.put("/providers/detect-mode")
|
||||
def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require_admin_dep)):
|
||||
from detect_learning import set_detect_operating_mode
|
||||
|
|
@ -243,12 +249,45 @@ def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require
|
|||
status_code=400,
|
||||
detail={
|
||||
"code": "invalid_detect_operating_mode",
|
||||
"message": "Detect-Modus muss semantic oder learning sein.",
|
||||
"message": "Detect-Modus muss student, learning_full, learning_reduced, learning_manual oder bypass sein.",
|
||||
},
|
||||
)
|
||||
return provider_public_status()
|
||||
|
||||
|
||||
@router.put("/providers/local-ner")
|
||||
def admin_set_local_ner(body: LocalNerWrite, session: dict = Depends(require_admin_dep)):
|
||||
from detect_learning import set_local_ner_settings
|
||||
from provider_settings import public_status as provider_public_status
|
||||
|
||||
try:
|
||||
set_local_ner_settings(
|
||||
enabled=body.enabled,
|
||||
url=body.url,
|
||||
passage_profile_id=body.passage_profile_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
code = str(exc)
|
||||
if code == "gliner_url_not_local":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": code,
|
||||
"message": "GLiNER-URL muss in der lokalen Trusted Zone liegen.",
|
||||
},
|
||||
) from exc
|
||||
if code == "passage_profile_missing":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": code, "message": "Das Klärungs-Profil fehlt."},
|
||||
) from exc
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "invalid_local_ner", "message": "Lokale NER-Einstellung ungültig."},
|
||||
) from exc
|
||||
return provider_public_status()
|
||||
|
||||
|
||||
@router.get("/providers")
|
||||
def admin_providers(session: dict = Depends(require_admin_dep)):
|
||||
return public_status()
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from dialogue_store import StoreError, delete_conversation, get_conversation, li
|
|||
from dialogue_turn import continue_turn, run_turn, visible_for_role
|
||||
from privacy_gateway import GatewayRequest, inspect, public_error_detail
|
||||
from engine import EngineError
|
||||
from journal_generate import generate_draft
|
||||
from journal_generate import continue_journal_generate, generate_draft
|
||||
from journal_generation_policy import settings_payload
|
||||
from journal_opening import maybe_open_journal_conversation
|
||||
from journal_policy import PolicyError
|
||||
|
|
@ -353,6 +353,22 @@ def generate(journal_day_id: str, body: GenerateWrite, session: dict = Depends(r
|
|||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/days/{journal_day_id}/generate/review")
|
||||
def generate_review(journal_day_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
continue_journal_generate(
|
||||
session["profile_id"],
|
||||
journal_day_id,
|
||||
body.review_id,
|
||||
body.decisions,
|
||||
),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, PolicyError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries")
|
||||
def write_entry(body: EntryWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ UNIT_MODULES = frozenset(
|
|||
"test_journal_shape",
|
||||
"test_journal_body",
|
||||
"test_writing_profile",
|
||||
"test_gliner_client",
|
||||
"test_detect_pipeline",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,12 +18,27 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
|
|||
from fastapi.testclient import TestClient
|
||||
from detect_learning import (
|
||||
cue_for_mention,
|
||||
decision_for_mention,
|
||||
excerpt_view,
|
||||
get_cue_decision,
|
||||
get_detect_operating_mode,
|
||||
get_sense,
|
||||
merge_gliner_candidates,
|
||||
passage_provider,
|
||||
record_cue,
|
||||
record_sense,
|
||||
set_detect_operating_mode,
|
||||
try_local_passage_decision,
|
||||
)
|
||||
from detect_pipeline import (
|
||||
MentionSpan,
|
||||
compact_diff_counters,
|
||||
compare_mentions,
|
||||
map_entity_type,
|
||||
prepare_detect_pipeline,
|
||||
same_mention,
|
||||
)
|
||||
from providers import ChatResult, ProviderConfig
|
||||
from dialogue_turn import visible_for_role
|
||||
from identity_store import list_confirmed_identities
|
||||
from main import app
|
||||
|
|
@ -75,8 +90,28 @@ def main() -> None:
|
|||
|
||||
providers = client.get("/api/admin/providers", headers=headers)
|
||||
expect(providers.status_code == 200, "providers status")
|
||||
expect(providers.json()["detect_operating_mode"] == "semantic", "default detect mode is semantic")
|
||||
expect(get_detect_operating_mode() == "semantic", "store default is semantic")
|
||||
expect(providers.json()["detect_operating_mode"] == "bypass", "fake-env default is bypass")
|
||||
expect(get_detect_operating_mode() == "bypass", "store default in tests is bypass")
|
||||
ner = providers.json().get("local_ner") or {}
|
||||
expect(ner.get("enabled") is False, "GLiNER is off by default")
|
||||
detect_before = next(item for item in providers.json()["roles"] if item["role"] == "detect")
|
||||
remote_gliner = client.put(
|
||||
"/api/admin/providers/local-ner",
|
||||
headers=headers,
|
||||
json={"url": "https://openrouter.ai/api/v1"},
|
||||
)
|
||||
expect(remote_gliner.status_code == 400, "GLiNER URL must stay local")
|
||||
enabled = client.put(
|
||||
"/api/admin/providers/local-ner",
|
||||
headers=headers,
|
||||
json={"enabled": True, "url": "http://192.168.2.144:8090"},
|
||||
)
|
||||
status_ok(enabled, "enable local GLiNER")
|
||||
expect(enabled.json()["local_ner"]["enabled"] is True, "GLiNER enabled flag persisted")
|
||||
expect(enabled.json()["local_ner"]["url"] == "http://192.168.2.144:8090", "GLiNER LAN URL persisted")
|
||||
detect_after = next(item for item in enabled.json()["roles"] if item["role"] == "detect")
|
||||
expect(detect_after.get("url") == detect_before.get("url"), "OpenRouter detect URL is unchanged")
|
||||
client.put("/api/admin/providers/local-ner", headers=headers, json={"enabled": False})
|
||||
|
||||
bad = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "wordlist"})
|
||||
expect(bad.status_code == 400, "invalid detect mode is rejected")
|
||||
|
|
@ -87,13 +122,13 @@ def main() -> None:
|
|||
headers=headers,
|
||||
json={"body": "Anna kam vorbei."},
|
||||
)
|
||||
status_ok(semantic_turn, "semantic turn")
|
||||
expect(not semantic_turn.json().get("pending_mask_review"), "semantic mode does not pause for review")
|
||||
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "semantic turn gets an assistant")
|
||||
status_ok(semantic_turn, "bypass turn")
|
||||
expect(not semantic_turn.json().get("pending_mask_review"), "bypass mode does not pause for review")
|
||||
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "bypass turn gets an assistant")
|
||||
|
||||
switched = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning"})
|
||||
status_ok(switched, "switch to learning")
|
||||
expect(switched.json()["detect_operating_mode"] == "learning", "learning mode persisted")
|
||||
status_ok(switched, "switch to learning alias")
|
||||
expect(switched.json()["detect_operating_mode"] == "learning_full", "learning alias stores learning_full")
|
||||
|
||||
conv = open_conv(client, headers)
|
||||
paused = client.post(
|
||||
|
|
@ -174,7 +209,11 @@ def main() -> None:
|
|||
json={"body": "Hanna aß mit uns."},
|
||||
)
|
||||
status_ok(hanna_again, "second Hanna turn")
|
||||
expect(not hanna_again.json().get("pending_mask_review"), "known non-identity skips the popup")
|
||||
hanna_conflict = hanna_again.json().get("pending_mask_review") or {}
|
||||
expect(
|
||||
bool(hanna_conflict.get("id")),
|
||||
"teacher PERSON versus stored not_identity cue stays a visible type conflict",
|
||||
)
|
||||
|
||||
record_sense(profile_id, "Clarissa", identity=True)
|
||||
record_sense(profile_id, "Clarissa", identity=False)
|
||||
|
|
@ -311,8 +350,89 @@ def main() -> None:
|
|||
status_ok(sushi_again, "repeat homonym sentence")
|
||||
expect(not sushi_again.json().get("pending_mask_review"), "same reviewed sentence does not re-ask")
|
||||
|
||||
dish = "Ich aß Sushi."
|
||||
dish_at = dish.index("Sushi")
|
||||
food_spans = [{"text": "Sushi", "start": dish_at, "end": dish_at + 5, "label": "food", "score": 0.8}]
|
||||
food_cands, _maps = merge_gliner_candidates(profile_id, [], [], dish, dish, food_spans)
|
||||
expect(
|
||||
any(item.get("text") == "Sushi" and item.get("suggested") == "not_identity" for item in food_cands),
|
||||
"GLiNER food span suggests not_identity",
|
||||
)
|
||||
person_line = "Ich war mit meiner Frau Sushi unterwegs."
|
||||
person_at = person_line.index("Sushi")
|
||||
gliner_miss = [{"text": "Sushi", "start": person_at, "end": person_at + 5, "label": "food", "score": 0.9}]
|
||||
merged, _maps = merge_gliner_candidates(profile_id, [], [], person_line, person_line, gliner_miss)
|
||||
expect(
|
||||
not any(item.get("suggested") == "not_identity" for item in merged),
|
||||
"GLiNER food does not override Frau attachment",
|
||||
)
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", person_line, person_at, person_at + 5, person_line) == "identity",
|
||||
"learned Frau cue wins over GLiNER food",
|
||||
)
|
||||
dish_line = "Ich aß Sushi."
|
||||
dish_pos = dish_line.index("Sushi")
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", dish_line, dish_pos, dish_pos + 5, dish_line) == "ask",
|
||||
"Tag-21: identity hits plus new cue stay visible",
|
||||
)
|
||||
person = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
|
||||
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
|
||||
expect(same_mention(person, food), "same offsets are the same mention")
|
||||
wrapped = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
|
||||
expect(same_mention(person, wrapped), "contained Sushi matches Frau Sushi")
|
||||
expect(map_entity_type("person") == "PERSON", "GLiNER person maps to PERSON")
|
||||
review, counters = compare_mentions(profile_id, dish_line, [person], [food], dish_line)
|
||||
expect(counters["diff_type_conflict"] >= 1, "FOOD versus PERSON is a type conflict")
|
||||
expect(any(item.get("reason") == "type_conflict" for item in review), "type conflict is offered")
|
||||
compact = compact_diff_counters(counters)
|
||||
dumped = str(compact)
|
||||
expect("Sushi" not in dumped and "Frau" not in dumped, "compact diff counters carry no plaintext")
|
||||
expect(set(compact) == {
|
||||
"diff_agree",
|
||||
"diff_only_student",
|
||||
"diff_only_teacher",
|
||||
"diff_type_conflict",
|
||||
"diff_new_identity",
|
||||
"diff_cue_reinforced",
|
||||
}, "compact counters are the agreed keys")
|
||||
set_detect_operating_mode("learning_reduced")
|
||||
expect(get_detect_operating_mode() == "learning_reduced", "stage 2 mode persists")
|
||||
set_detect_operating_mode("learning_manual")
|
||||
expect(get_detect_operating_mode() == "learning_manual", "manual mode persists")
|
||||
set_detect_operating_mode("student")
|
||||
pipe = prepare_detect_pipeline(profile_id, "Anna kam vorbei.", "Anna kam vorbei.")
|
||||
expect(pipe.mode == "student", "student path is selectable")
|
||||
expect(not pipe.need_review, "zielpfad does not open a teacher popup")
|
||||
record_sense(profile_id, "Sushi", identity=True)
|
||||
record_cue(profile_id, "Sushi", "frau", "identity", "meine Frau Sushi")
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", "Ich aß Sushi.", dish_pos, dish_pos + 5) == "ask",
|
||||
"Tag-21 remains ask after more identity hits",
|
||||
)
|
||||
|
||||
fake_passage = ProviderConfig(
|
||||
role="passage",
|
||||
name="ollama",
|
||||
mode="http",
|
||||
url="http://192.168.2.144:11434/v1/chat/completions",
|
||||
model="phi3:mini",
|
||||
key="",
|
||||
local=True,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
with patch("detect_learning.passage_provider", return_value=fake_passage):
|
||||
with patch("providers.complete_chat", return_value=ChatResult(content='{"decision":"not_identity"}')) as chat:
|
||||
decided = try_local_passage_decision("Rohan Sushi essen", "Sushi")
|
||||
expect(decided == "not_identity", "passage profile can decide without local Detect")
|
||||
expect(chat.called, "passage uses complete_chat")
|
||||
expect(passage_provider() is None, "no passage profile means no silent detect fallback")
|
||||
|
||||
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
|
||||
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
|
||||
expect(back.json()["detect_operating_mode"] == "bypass", "semantic alias stores bypass")
|
||||
full = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning_full"})
|
||||
expect(full.json()["detect_operating_mode"] == "learning_full", "learning_full is accepted")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
68
backend/tests/test_detect_pipeline.py
Normal file
68
backend/tests/test_detect_pipeline.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
"""Unit tests for student/teacher comparison. No database."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from detect_learning import canonicalize_detect_mode, decision_for_mention
|
||||
from detect_pipeline import (
|
||||
MentionSpan,
|
||||
compact_diff_counters,
|
||||
map_entity_type,
|
||||
protects,
|
||||
same_mention,
|
||||
types_equivalent,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(canonicalize_detect_mode("learning") == "learning_full", "learning aliases to learning_full")
|
||||
expect(canonicalize_detect_mode("semantic") == "bypass", "semantic aliases to bypass")
|
||||
expect(canonicalize_detect_mode("wordlist") is None, "unknown mode is rejected")
|
||||
expect(map_entity_type("person") == "PERSON", "person maps to PERSON")
|
||||
expect(map_entity_type("place") == "PLACE", "place maps to PLACE")
|
||||
expect(protects("PERSON") and not protects("FOOD"), "PERSON protects, FOOD does not")
|
||||
expect(types_equivalent("PERSON", "PLACE"), "similar protectable types are equivalent")
|
||||
expect(not types_equivalent("FOOD", "PERSON"), "FOOD versus PERSON is never equivalent")
|
||||
inner = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
|
||||
outer = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
|
||||
expect(same_mention(inner, outer), "contained span is the same mention")
|
||||
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
|
||||
expect(same_mention(inner, food), "same offsets stay one mention even when types differ")
|
||||
compact = compact_diff_counters({"diff_agree": 2, "diff_type_conflict": 1, "label": "Sushi"})
|
||||
expect(compact["diff_agree"] == 2 and compact["diff_type_conflict"] == 1, "known counters are kept")
|
||||
expect("label" not in compact and "Sushi" not in str(compact), "compact counters drop plaintext")
|
||||
with patch("detect_learning.get_cue_decision", return_value=None), patch(
|
||||
"detect_learning.get_sense",
|
||||
return_value={"identity_hits": 3, "non_identity_hits": 0, "ambiguous": False},
|
||||
), patch("detect_learning.cue_for_mention", return_value="aß"):
|
||||
expect(
|
||||
decision_for_mention("p", "Sushi", "Ich aß Sushi.", 7, 12) == "ask",
|
||||
"Tag-21: identity-only sense does not lock a new cue",
|
||||
)
|
||||
with patch("detect_learning.get_cue_decision", return_value="identity"), patch(
|
||||
"detect_learning.cue_for_mention", return_value="frau"
|
||||
):
|
||||
expect(
|
||||
decision_for_mention("p", "Sushi", "meine Frau Sushi", 11, 16) == "identity",
|
||||
"stored cue remains binding",
|
||||
)
|
||||
print("detect pipeline unit ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -89,7 +89,7 @@ def main() -> None:
|
|||
expect(set(roles) == {"generate", "detect"}, "two provider roles")
|
||||
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
|
||||
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
|
||||
expect(providers.json().get("detect_operating_mode") == "semantic", "detect mode defaults to semantic")
|
||||
expect(providers.json().get("detect_operating_mode") == "learning_full", "detect mode defaults to learning stage 1")
|
||||
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
|
||||
expect("sk-" not in providers.text, "provider status has no secret")
|
||||
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")
|
||||
|
|
|
|||
54
backend/tests/test_gliner_client.py
Normal file
54
backend/tests/test_gliner_client.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""GLiNER client: local URL only, fail-soft, no request text in logs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||
|
||||
from gliner_client import DEFAULT_URL, extract_entities, health, normalize_base_url
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code: int, payload):
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(normalize_base_url("http://192.168.2.144:8090/") == DEFAULT_URL, "default GLiNER URL is normalized")
|
||||
expect(extract_entities("Anna", url="https://openrouter.ai/api/v1") is None, "OpenRouter URL is rejected")
|
||||
expect(health("https://openrouter.ai").get("reason") == "not_local", "health rejects non-local URL")
|
||||
|
||||
with patch("gliner_client.httpx.post", return_value=_Resp(200, {"spans": [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}]})):
|
||||
spans = extract_entities("Anna kam.", url="http://192.168.2.144:8090")
|
||||
expect(spans == [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}], "extract returns spans")
|
||||
|
||||
with patch("gliner_client.httpx.post", side_effect=httpx.ConnectError("down")):
|
||||
expect(extract_entities("Anna kam.", url="http://192.168.2.144:8090") is None, "extract is fail-soft on errors")
|
||||
|
||||
with patch("gliner_client.httpx.get", return_value=_Resp(200, {"status": "ok", "model": "urchade/gliner_multi-v2.1", "offline": True})):
|
||||
ping = health("http://192.168.2.144:8090")
|
||||
expect(ping.get("ok") is True and ping.get("offline") is True, "health reports ready offline model")
|
||||
print("gliner client: OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -31,7 +31,7 @@ def expect(ok: bool, message: str) -> None:
|
|||
def main() -> None:
|
||||
expect(is_local_url("http://127.0.0.1:11434/v1/chat/completions"), "loopback is local")
|
||||
expect(is_local_url("http://localhost:11434/v1/chat/completions"), "localhost is local")
|
||||
expect(is_local_url("http://192.168.2.144:11434/v1/chat/completions"), "RFC1918 Ollama host is local")
|
||||
expect(is_local_url("http://192.168.2.144:8090"), "RFC1918 GLiNER host is local")
|
||||
expect(is_local_url("http://10.0.0.8:11434/v1/chat/completions"), "10/8 is local")
|
||||
expect(is_local_url("http://ollama:11434/v1/chat/completions"), "compose hostname ollama is local")
|
||||
expect(not is_local_url("https://openrouter.ai/api/v1/chat/completions"), "OpenRouter is not local")
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ Kanonisches Home der Fit-Gap-Analyse ist `mvp_stand_und_abgleich.md`. Es ersetzt
|
|||
3. `self_model_and_lived_experience.md`
|
||||
4. `memory_and_context.md`
|
||||
5. das konkret betroffene Technik- oder Integrationskapitel
|
||||
6. bei Detect-Selbstlernen (Konzept, nicht Code-Stand): `docs/work_orders/detect_self_learning_pipeline.md`
|
||||
|
||||
## 4. Technische Architektur
|
||||
|
||||
|
|
@ -103,7 +104,7 @@ Sie übernimmt den Produktrahmen **weitgehend von Mitai**. Datenschutz, Privacy
|
|||
|
||||
Technische Umsetzung des gebauten Journal-Slices: `../technical/mvp_implementation.md`. Fit-Gap bleibt fachlich: `mvp_stand_und_abgleich.md`.
|
||||
|
||||
Arbeitsaufträge (kein Ersatz kanonischer Kapitel): `docs/work_orders/` – Laptop-Abschluss, Heim-Setup, Debug-Export, Dialog/Memory-Slice. Slice-2-Konzeptbrief derzeit `Idea_seconmd_sclide.md`.
|
||||
Arbeitsaufträge (kein Ersatz kanonischer Kapitel): `docs/work_orders/` – Laptop-Abschluss, Heim-Setup, Debug-Export, Dialog/Memory-Slice, Selbstlern-Detect-Pipeline (`detect_self_learning_pipeline.md`, Konzept 2026-09-11). Slice-2-Konzeptbrief derzeit `Idea_seconmd_sclide.md`.
|
||||
|
||||
## 5. Ladeprinzip
|
||||
|
||||
|
|
|
|||
|
|
@ -719,7 +719,13 @@ Additiv. Technische Abbildung: `../technical/privacy_gateway.md` §9.7.
|
|||
|
||||
**Entschieden (Übergang):** Eine Mini-Passage an ein internes Modell geht nur bei bereits mehrdeutiger Schreibweise und nur an ein lokales Detect. OpenRouter sieht diese Passage nicht. Fehlt das lokale Modell, bleibt das Popup.
|
||||
|
||||
**Additiv 2026-09-10:** Eine bestätigte Nennung wird nicht erneut gefragt, wenn derselbe lokale Kontext wiederkehrt. Enge Verwandtschaftsbindung in der Nominalphrase (auch Apposition: `Sushi, meine Frau`) ist ein Hinweis, nicht die ganze Wahrheit. Der gespeicherte Ausschnitt ist der Kontext für eine lokale Vorentscheidung. Ein späteres lokales GLiNER/Detect sieht diese Mini-Passage plus bestätigte Beispiele; OpenRouter sieht sie nicht. Unbekannte Kontexte bleiben prüfpflichtig. Das ist kein Wortlisten-Editor.
|
||||
**Additiv 2026-09-10:** Eine bestätigte Nennung wird nicht erneut gefragt, wenn derselbe lokale Kontext wiederkehrt. Enge Verwandtschaftsbindung in der Nominalphrase (auch Apposition: `Sushi, meine Frau`) ist ein Hinweis, nicht die ganze Wahrheit. Der gespeicherte Ausschnitt ist der Kontext für eine lokale Vorentscheidung.
|
||||
|
||||
**Additiv 2026-09-11 (GLiNER-Vorstufe):** Im Lernmodus darf die LAN-Instanz `http://192.168.2.144:8090` den aktuellen Nutzersatz sehen (`POST /v1/extract`). Gelernte Sinne/Cues bleiben führend. Unsichere Nennungen gehen als Mini-Passage an ein im Admin gewähltes LLM-Profil (Ollama oder OpenRouter), nicht an die Detect-Stufe. OpenRouter-Maskierung bleibt vollständig konfigurierbar und ist in diesem Slice weiter der Egress-Detect. GLiNER ist kein alleiniges Detect und keine Wortliste. Fail-soft, wenn der Dienst fehlt.
|
||||
|
||||
**Additiv 2026-09-11 (Selbstlern-Pipeline, Konzept):** Zielbild mit Schülerpfad vor GLiNER/Lehrer und zwei Lernstufen: `../../work_orders/detect_self_learning_pipeline.md`. Das ersetzt den laufenden Lernmodus-Code nicht. Offene Punkte dort §6 vor der Umsetzung klären.
|
||||
|
||||
**Additiv 2026-09-11 (Selbstlern-Pipeline, §6 geschlossen):** Start ist Lernmodus Stufe 1: Schüler (Cues, Verwandtschaft, Homonym-Mini-LLM nur bei beiden Sinnen, GLiNER) parallel zum Lehrer auf dem unveränderten Nutzersatz. Admin: Zielpfad, Lernen Stufe 1/2/manuell, Bypass (alter Voll-Detect, mittelfristig weg). Default nach Umbau: Stufe 1. Neuer Cue bei nur Identity-Hits bleibt sichtbar (Sushi-Tag-21). Cue gilt pro Vorkommen. Lehrer-Miss eines bestätigten Cues ohne Meldung verlernt nicht. FOOD gegen PERSON ist immer Diff. Diagnose nur Zähler ohne Klartext. Gilt für alle persönlichen Detect-Aufrufe einschließlich Journalentwurf. GLiNER bleibt fail-soft im Lernmodus (dann manuelle Review); außerhalb Lernmodus Bestätigung vor öffentlichem Klartext-Egress. Gateway nicht abschaltbar.
|
||||
|
||||
**Nicht:** Gateway abschalten, Detect-Treffer auto-speichern, `Sushi_`/`Sushi+` im Nutzertext, Pattern-Wortliste als Wahrheit.
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,10 @@ Nicht übernehmen: Mitai-Admin für Körpertarife, Coupons, Training Types als K
|
|||
|
||||
**Additiv 2026-09-08 (Detect-Lernmodus):** `PUT /api/admin/providers/detect-mode` mit `semantic` | `learning`. Statusfeld `detect_operating_mode`. Keine Wortlisten-Seite. Identitäten dürfen ein Badge „mehrdeutig“ aus Dialog-Sinnen zeigen. Dialog und Journal-Gespräch pausieren im Lernmodus mit Bestätigungs-Popup; Journal-Generate nicht.
|
||||
|
||||
**Additiv 2026-09-11 (GLiNER-Vorstufe):** `PUT /api/admin/providers/local-ner` mit `enabled`, `url`, `passage_profile_id`. Status `local_ner`. Unabhängig von der Detect-URL. GLiNER nur lokale Trusted-Zone. Klärung unsicherer Nennungen über ein bestehendes LLM-Profil.
|
||||
|
||||
**Additiv 2026-09-11 (Selbstlern-Pipeline):** `PUT /api/admin/providers/detect-mode` nimmt `student` | `learning_full` | `learning_reduced` | `learning_manual` | `bypass` (Aliase `learning`, `semantic`). Compact-Diagnose darf Diff-Zähler ohne Klartext zeigen, keine Langeweile-Kennzahl. Stufe wechselt nur der Admin. Journalentwurf nutzt dieselbe Bestätigungsfläche.
|
||||
|
||||
## 4.2 Implementierungsstand (Dialog-Testspur)
|
||||
|
||||
**Additiv 2026-08-26:** Lokale Identitätsregistry unter `/admin/identities`. Detect-Vorschläge sind unbestätigt. Compact-Diagnose enthält Detect-Abdeckung, Chunks, Kosten und Laufzeit, aber keine Labels.
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ Kanonisches Home **wie** der Slice gebaut ist: `mvp_implementation.md`. Fachlich
|
|||
6. `../../work_orders/home_environment_setup.md`
|
||||
7. fachlich: `../functional/guardrails.md` (Datenklassen beim Kopieren von Backup und `.env`)
|
||||
|
||||
Arbeitsaufträge liegen unter `docs/work_orders/`. Sie ersetzen keine kanonischen Kapitel. Betrieb auf dem Pi: `docs/DEPLOYMENT.md`.
|
||||
Arbeitsaufträge liegen unter `docs/work_orders/`. Sie ersetzen keine kanonischen Kapitel. Betrieb auf dem Pi: `docs/DEPLOYMENT.md`. Detect-Selbstlernen (Konzept 2026-09-11, Umsetzung ausstehend): `../../work_orders/detect_self_learning_pipeline.md`.
|
||||
|
||||
## 4. Ladeprinzip
|
||||
|
||||
|
|
|
|||
|
|
@ -255,6 +255,8 @@ Operator-Übergang bis Ollama: `KANSHO_ALLOW_REMOTE_DETECT=1` (Compose `.env`, C
|
|||
|
||||
Status in `GET /api/admin/providers`: `remote_plaintext_allowed`, `remote_plaintext_reason` (`non_production` | `operator_override` | `blocked`), `profiles`, `detect_operating_mode` (`semantic` | `learning`). Modellauswahl: `GET /api/admin/providers/models` (OpenRouter-Katalog oder Ollama `/api/tags`, soft-fail).
|
||||
|
||||
**Additiv 2026-09-11:** `detect_operating_mode` erweitert, siehe §9.7. Die Werte `semantic` und `learning` bleiben als Alias lesbar (`bypass` / `learning_full`).
|
||||
|
||||
## 9.7 Übergang: Detect-Lernmodus (2026-09-08)
|
||||
|
||||
Additiv. Fachliches Home: `../functional/guardrails.md` §22.4. Ersetzt weder semantische Detection noch die bestätigte Registry.
|
||||
|
|
@ -265,7 +267,13 @@ Im Lernmodus untersucht Detect weiterhin den vollen gerenderten Generate-Egress.
|
|||
|
||||
Es gibt keine separat zu pflegende Doppeldeutigkeitsliste. Mehrdeutigkeit entsteht, wenn dieselbe Schreibweise beide Sinne hat. Nur dann darf ein **lokales** Detect-Modell eine Mini-Passage (Ausschnitt um die Nennung) entscheiden. Fehlt ein lokales Modell oder ist die Antwort unbrauchbar, bleibt das Popup. OpenRouter erhält diese Passage nicht.
|
||||
|
||||
**Additiv 2026-09-10:** Bestätigungen speichern den lokalen Cue und den Ausschnitt in `label_sense_cues`. Cue ist die enge Bindung in der Phrase (`Frau Sushi`, `Sushi, meine Frau`), sonst das Wort unmittelbar davor — nicht jedes Verwandtschaftswort irgendwo im Satz, damit Homonyme (`Frau Sushi` / Speise) getrennt bleiben. Der Ausschnitt geht nur an ein **lokales** Detect/GLiNER als Mini-Passage plus bestätigte Beispiele; das ist die Vorentscheidung. GLiNER ist dafür der bevorzugte lokale Weg, in diesem Slice noch nicht verdrahtet. Fehlt das lokale Modell, bleibt das Popup. OpenRouter erhält diese Passage nicht.
|
||||
**Additiv 2026-09-10:** Bestätigungen speichern den lokalen Cue und den Ausschnitt in `label_sense_cues`. Cue ist die enge Bindung in der Phrase (`Frau Sushi`, `Sushi, meine Frau`), sonst das Wort unmittelbar davor — nicht jedes Verwandtschaftswort irgendwo im Satz, damit Homonyme (`Frau Sushi` / Speise) getrennt bleiben.
|
||||
|
||||
**Additiv 2026-09-11:** Lernmodus kann LAN-GLiNER (`192.168.2.144:8090`, `POST /v1/extract`) auf den Nutzersatz anwenden. Settings: `gliner_enabled`, `gliner_url`, `passage_profile_id` in `app_settings`, nicht in `provider_settings.detect`. Cues überschreiben GLiNER. Mini-Passagen gehen an das gewählte LLM-Profil. Detect-OpenRouter bleibt die Egress-Maskierung. Kein Klartext in GLiNER-Logs seitens Kanshō. Tests: `backend/tests/test_gliner_client.py`, `backend/tests/test_detect_learning.py`.
|
||||
|
||||
**Additiv 2026-09-11 (Selbstlern-Pipeline, Konzept):** Reihenfolge umkehren und zwei Lernstufen: `../../work_orders/detect_self_learning_pipeline.md`. Ist-Code bleibt OpenRouter-Detect zuerst, bis die Umsetzung nach Finalisierung von §6 dort folgt.
|
||||
|
||||
**Additiv 2026-09-11 (Selbstlern-Pipeline, Umsetzung nach §6):** `detect_operating_mode` ist `student` | `learning_full` | `learning_reduced` | `learning_manual` | `bypass`. Aliase: `learning` → `learning_full`, `semantic` → `bypass`. Default im Produkt: `learning_full`. Schülerpfad in `detect_pipeline.run_student_path`. Lehrer Stufe 1 = unveränderter Nutzersatz, Stufe 2 = ohne cue-sichere Oberflächenformen. Vergleich nur User-Spans; Compact-Zähler `diff_agree`, `diff_only_student`, `diff_only_teacher`, `diff_type_conflict`, `diff_new_identity`, `diff_cue_reinforced` ohne Labels. Journalentwurf pausiert über `POST /api/journal/days/{id}/generate/review` und dieselbe Review-UI. Bypass bleibt der alte Detect auf dem gerenderten Prompt.
|
||||
|
||||
Nach der Bestätigung läuft Generate mit den geprüften Mappings (`precomputed_learning_review`), ohne zweiten Detect-Pass. Compact-Diagnose enthält weiterhin keine Klartextlabels. Tests: `backend/tests/test_detect_learning.py`.
|
||||
|
||||
|
|
|
|||
177
docs/work_orders/detect_self_learning_pipeline.md
Normal file
177
docs/work_orders/detect_self_learning_pipeline.md
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
---
|
||||
title: "Kanshō – Selbstlernende Detect-Pipeline (Konzept und Anforderung)"
|
||||
status: "Konzept / Arbeitsauftrag — §6 finalisiert 2026-09-11"
|
||||
date: "2026-09-11"
|
||||
document_role: "Cursor Implementation Brief / Concept Record"
|
||||
canonical_functional: "docs/architecture/functional/guardrails.md §22.4"
|
||||
canonical_technical: "docs/architecture/technical/privacy_gateway.md §9.7"
|
||||
---
|
||||
|
||||
# Selbstlernende Detect-Pipeline
|
||||
|
||||
Dieses Dokument ist das Home für das **Zielbild 2026-09-11**. Es ersetzt weder `guardrails.md` noch `privacy_gateway.md`. Statuswörter ernst nehmen. §6 wurde am 2026-09-11 mit dem Nutzer im Planmodus geschlossen; die Umsetzung folgt diesen Entscheidungen.
|
||||
|
||||
Herkunft: explizite Nutzeranforderung und gemeinsame Bewertung in der Konzeptsession 2026-09-11.
|
||||
|
||||
## 1. Ziel
|
||||
|
||||
Ein **selbstlernender Schutz** pro Profil, der über Zeit immer weniger persönliche Klarformen an ein externes LLM gibt und dabei verlässlicher wird.
|
||||
|
||||
GLiNER findet neue Spannen. Kanshō entscheidet und merkt (Sinne, Cues, Registry). OpenRouter ist Lehrer und Lückenfüller, nicht Alltag und nicht Identitätsquelle. GLiNER-Gewichte werden zur Laufzeit nicht nachtrainiert; ein gemeinsames Jinkendo-NER-Projekt ist ein späterer, getrennter Dienst (stateless, keine gemeinsame Identität).
|
||||
|
||||
Leitbeispiel: Lange Zeit bedeutet „Sushi“ die Partnerin. Am Tag 21 ist „Sushi“ Essen. Ein Lexem-Lock auf Identität und ein Lehrer, der nur noch Platzhalter sieht, machen das Essen unsichtbar. Das darf nicht passieren.
|
||||
|
||||
## 2. Entscheidungsstand
|
||||
|
||||
### Entschieden
|
||||
|
||||
- Identität bleibt lokal. Klasse-A-Mapping, Klarnamen und Registry gehen nicht an Provider.
|
||||
- Keine Wortlisten-UI. Sinne entstehen aus Bestätigungen (Identität / nicht schützenswert).
|
||||
- OpenRouter darf Identitäten nicht stillschweigend anlegen oder löschen. Der Mensch entscheidet über neue Identität und über Abweichungen.
|
||||
- Cue-Treffer gelten nur für **dieses Vorkommen**, nicht für die Schreibweise insgesamt.
|
||||
- Nur Identity-Hits ohne Non-Identity dürfen einen **neuen** Cue nicht festnageln (Sushi-Tag-21).
|
||||
- Schülerpfad-Reihenfolge: lokal cue-aufgelöste Vorkommen → unklare Homonyme als enger Ausschnitt an das Passage-LLM → GLiNER auf den Rest → Generate mit gebauten Mappings.
|
||||
- GLiNER sieht die Originalzeile mit übersprungenen, bereits aufgelösten Offsets, nicht den schon ersetzten Platzhaltertext.
|
||||
- Zwei Lernstufen. Umschalten ist eine Admin-Entscheidung, kein Autopilot.
|
||||
- **Stufe 1 (vollständig):** Kontrollaufruf auf den **unveränderten Nutzersatz** (keine Ersetzungen), parallel der Schülerpfad, Vergleich, bei Uneinigkeit oder neuer Identität Popup, dann lernen.
|
||||
- **Stufe 2:** Lehrer sieht keine **cue-sicheren** Vorkommen mehr. Dieselbe Vergleichslogik. Am Anfang fehleranfälliger; deshalb nach Stufe 1, nicht als Start.
|
||||
- Lehrer-Slice ist der Nutzersatz, nicht der gerenderte Detect-Prompt (kein Mitnehmen von Biografie/Kontext, den GLiNER nicht sieht).
|
||||
- Übereinstimmung plus bereits bestätigtes Label: Cue verstärken, kein Popup.
|
||||
- GLiNER nicht als alleiniges Detect in die Detect-URL schreiben. Detect-OpenRouter bleibt konfigurierbar.
|
||||
- Kein Gateway-Bypass, kein Feature-Flag zum Abschalten der Maskierung.
|
||||
- Bestehenden GLiNER-Client, Local-NER-Settings, Cues und Review-UI wiederverwenden, nicht wegwerfen; **umverdrahten**.
|
||||
- **Startbetrieb:** Lernmodus Stufe 1. Schüler (Cues → Verwandtschaft → Homonym-Mini-LLM bei beiden Sinnen → GLiNER) und Lehrer (unveränderter Nutzersatz) parallel. Diff zum Nutzer. Geprüftes Ergebnis maskiert Generate.
|
||||
- **Admin:** Zielpfad (`student`) + Lernen automatisch Stufe 1 (`learning_full`) + Lernen automatisch Stufe 2 (`learning_reduced`) + Lernen manuell (`learning_manual`) + Bypass (`bypass`, alter OpenRouter-Voll-Detect). Bypass entfällt mittelfristig. Default nach Umbau: `learning_full`. Aliase: `learning` → `learning_full`, `semantic` → `bypass`.
|
||||
- **Stufe 2** ist im ersten Schnitt vollständig (Lehrer ohne cue-sichere Oberflächenformen).
|
||||
- **Lehrer weg:** alle Schüler-/GLiNER-Funde zur Bestätigung (gleicher Weg wie manuell).
|
||||
- **GLiNER weg:** im Lernmodus manuelle Review nach lokaler Vorauswahl. Außerhalb Lernmodus: Nutzer bestätigt Klartext-Egress an das öffentliche LLM.
|
||||
- **Verwandtschaft** bleibt eigener lokaler Schüler-Schritt vor GLiNER.
|
||||
- **Vergleich:** gleiche Nennung = gleicher Wortlaut an derselben Stelle oder eine Spanne vollständig in der anderen. Typen `person→PERSON` usw.; ähnliche schützenswerte Typen gleich; FOOD gegen PERSON immer Diff. Nur bei abweichendem Schutz fragen; gleicher Schutz, anderer Zuschnitt → längere Spanne. Nutzer entscheidet Speise/Person; Cue nicht still ändern; Lehrer-Miss ohne Meldung → kein Popup, Cue bleibt.
|
||||
- **Diagnose:** Zähler ohne Klartext (Einigkeit, nur Schüler, nur Lehrer, Typkonflikt, neue Identität, Cue verstärkt). Keine Langeweile-Kennzahl; Stufe nur per Schalter.
|
||||
- **Mini-LLM** nur wenn beide Sinne schon existieren, auch in Stufe 1.
|
||||
- **Geltungsbereich:** alle persönlichen Detect-Aufrufe (Security-Gateway), einschließlich Journalentwurf.
|
||||
|
||||
### Bevorzugte Richtung
|
||||
|
||||
- Nach ausreichend langweiliger Abweichung in Stufe 1: Stufe 2, später Betrieb ohne Kontrollaufruf (Schülerpfad allein).
|
||||
- Diagnostik zeigt Diff-Art (nur Schüler / nur Lehrer / Typkonflikt), ohne Klartextlabels in Compact-Exports.
|
||||
- Fragment-Ausschnitte so eng, dass fremde noch unmaskierte Identitäten nicht mitgehen.
|
||||
- Eigenes Jinkendo-Local-Detect als späteres LAN-Projekt (Text rein, Spannen raus, kein Speicher). Nicht in diesem Slice.
|
||||
|
||||
### Verworfen
|
||||
|
||||
- GLiNER oder OpenRouter als alleiniges Detect.
|
||||
- Lexemweites „einmal Person, immer Person“.
|
||||
- Lehrer auf bereits komplett vormaskiertem Text in Stufe 1.
|
||||
- Online-Fine-Tune von GLiNER auf Journaltext.
|
||||
- Gemeinsame Namensliste über Jinkendo-Produkte.
|
||||
- Detect-Treffer ohne Nutzer als aktive Identität speichern.
|
||||
|
||||
### Offen
|
||||
|
||||
Keine offenen Punkte aus der Planmodus-Runde 2026-09-11. Frühere §6-Fragen sind unter Entschieden festgehalten.
|
||||
|
||||
## 3. Ziel-Workflow
|
||||
|
||||
```
|
||||
Nutzersatz
|
||||
│
|
||||
├─ Schüler (immer, sobald dieser Pfad gilt)
|
||||
│ 1. Cue-aufgelöste Vorkommen lokal binden
|
||||
│ (Identität maskieren / Nicht-Identität sperren)
|
||||
│ 2. Unklare Homonyme: enger Ausschnitt → Passage-LLM
|
||||
│ 3. Rest → GLiNER (Skip bereits gebundener Spannen)
|
||||
│ 4. Mappings für Generate
|
||||
│
|
||||
└─ Lehrer (nur Lernmodus)
|
||||
Stufe 1: unveränderter Nutzersatz → OpenRouter-Detect
|
||||
Stufe 2: Nutzersatz ohne cue-sichere Oberflächenformen
|
||||
Vergleich Schüler ↔ Lehrer
|
||||
Uneinigkeit / neue Identität → Nutzer → Cue/Sinn
|
||||
Einigkeit + bekanntes Label → Cue verstärken
|
||||
```
|
||||
|
||||
Generate folgt auf geprüfte bzw. schülerseitig gebaute Mappings, ohne zweiten Voll-Detect.
|
||||
|
||||
## 4. Ist-Stand (Code, 2026-09-11)
|
||||
|
||||
Nicht mit dem Zielbild verwechseln. Älterer Stand vor der Umsetzung nach §6:
|
||||
|
||||
- `_learning_pause` hing OpenRouter-Detect an den **gerenderten** Prompt, dann Verwandtschaft, GLiNER, Cues, Passage, Popup.
|
||||
- `decision_for_mention` nagelte bei nur Identity-Hits jeden neuen Cue fest (Tag-21).
|
||||
- Admin: `semantic` | `learning`.
|
||||
|
||||
**Additiv nach Umsetzung §6:** Schülerpfad in `detect_pipeline.py`, Lehrer auf dem Nutzersatz, Admin-Modi `student` | `learning_full` | `learning_reduced` | `learning_manual` | `bypass`. Tabellen `label_senses`, `label_sense_cues`, `pending_mask_reviews` bleiben die Lernspeicher.
|
||||
|
||||
## 5. Anforderungen an die Umsetzung (nach Finalisierung)
|
||||
|
||||
1. Policy: `decision_for_mention` so ändern, dass ein neuer Cue bei nur einem Sinn **sichtbar bleibt** (nicht automatisch binden). Cue-Hit bleibt bindend.
|
||||
2. Schülerpfad als eigene, testbare Funktion; `_learning_pause` und später `semantic` hängen daran, sobald §6 das festlegt.
|
||||
3. Lehrer Stufe 1: Detect-Schema auf dem Nutzersatz, Offsets vergleichbar machen (GLiNER `person|place|…` ↔ Detect `PERSON|…`).
|
||||
4. Diff nur auf User-Spans; Lehrer-Miss eines bereits nutzerbestätigten Cues nicht automatisch verlernen.
|
||||
5. Popup über vorhandenes `MaskReviewPanel`, Inhalt = Diff und neue Identitätskandidaten, nicht jedes schon gelöste Cue-Vorkommen.
|
||||
6. Admin: zwei Lernstufen wählbar (konkretes Schaltermodell laut §6).
|
||||
7. Tests: Tag-21-Sushi (Identity-Hits, Cue `Frau` vs. neuer Cue „essen“); Diff-Fälle; kein Klartext in Compact-Diagnose; Local-URL-Policy für GLiNER bleibt.
|
||||
8. Docs additiv in `guardrails.md` §22.4, `privacy_gateway.md` §9.7, `admin_diagnostics.md`. Bestehende Absätze nicht kürzen. Dieses Work-Order-Dokument als Konzept-Home belassen, bis der Code dem Zielbild entspricht.
|
||||
9. Kein Commit/Push, unless explicitly requested.
|
||||
|
||||
Nicht in diesem Slice: Fine-Tune, neues NER-Repo, Wortlisten-Editor, Shared-Identity. Journal-Generate-Pause ist durch §6.9 **doch** im Schnitt (alle persönlichen Detect-Aufrufe).
|
||||
|
||||
## 6. Offene Punkte — geschlossen 2026-09-11
|
||||
|
||||
1. **Start ist Stufe 1 Lernen**, nicht Schüler-allein und nicht der alte `semantic`-Alltag. Schüler und Lehrer-Klartext (Nutzersatz) parallel, Diff zum Nutzer, dann Generate.
|
||||
2. **Admin:** Zielpfad + Lernen (automatisch 1 / automatisch 2 / manuell) + Bypass. Bypass mittelfristig weg. Default: `learning_full`.
|
||||
3. **Stufe 2 vollständig** im ersten Schnitt.
|
||||
4. **Lehrer weg** → manuelle Review aller GLiNER-/Schülerfunde. **GLiNER weg** → im Lernmodus manuell nach Vorauswahl; sonst Bestätigung vor öffentlichem Klartext-Egress.
|
||||
5. **Verwandtschaft bleibt** eigener lokaler Schritt vor GLiNER.
|
||||
6. **Vergleich:** enthaltene Spanne / gleicher Wortlaut; Mapping inkl. FOOD≠PERSON; nur abweichender Schutz fragt; Cue nicht still ändern; Lehrer-Miss ohne Meldung kein Popup.
|
||||
7. **Diagnose:** Zähler ohne Klartext, keine Langeweile-Kennzahl.
|
||||
8. **Mini-LLM** nur bei schon beiden Sinnen, auch in Stufe 1.
|
||||
9. **Alle persönlichen Detect-Aufrufe**, einschließlich Journalentwurf.
|
||||
|
||||
## 7. Context Bundle für die Umsetzungs-Session
|
||||
|
||||
1. Diese Datei.
|
||||
2. `docs/architecture/functional/documentation_index.md`
|
||||
3. `docs/architecture/functional/guardrails.md` (§22.4, Invarianten)
|
||||
4. `docs/architecture/technical/privacy_gateway.md` (§9.7)
|
||||
5. `docs/architecture/technical/admin_diagnostics.md`
|
||||
6. Code: `backend/detect_learning.py`, `backend/dialogue_turn.py`, `backend/gliner_client.py`, `frontend/src/pages/AdminProvidersPage.jsx`, `frontend/src/components/MaskReviewPanel.jsx`, `backend/tests/test_detect_learning.py`
|
||||
|
||||
Nicht den ganzen `docs/`-Ordner laden. Dialog-/MVP-Fachstand nicht stillschweigend festlegen.
|
||||
|
||||
## 8. Prompt für die neue Session
|
||||
|
||||
Den folgenden Block unverändert in eine **neue** Cursor-Session mit frischem Kontextfenster einfügen.
|
||||
|
||||
```
|
||||
Arbeite im Repo C:\Dev\Kansho. Sprache: Deutsch.
|
||||
|
||||
Zuerst in den Planmodus wechseln (SwitchMode → plan). Noch keinen Code schreiben, keine Policy umbauen, nicht committen.
|
||||
|
||||
Auftrag: Selbstlernende Detect-Pipeline gemäß
|
||||
docs/work_orders/detect_self_learning_pipeline.md
|
||||
|
||||
Das Dokument ist Konzept-Home. Es ersetzt guardrails.md / privacy_gateway.md nicht. Der Code hängt GLiNER und Cues noch hinter OpenRouter-Detect; decision_for_mention nagelt bei nur Identity-Hits jeden neuen Cue fest. Beides soll weg, aber erst nach Finalisierung der offenen Punkte.
|
||||
|
||||
Sofort laden (nur dieses Bundle):
|
||||
- docs/work_orders/detect_self_learning_pipeline.md (vollständig, besonders §2, §3, §6)
|
||||
- docs/architecture/functional/documentation_index.md
|
||||
- docs/architecture/functional/guardrails.md (§22.4 und Invarianten)
|
||||
- docs/architecture/technical/privacy_gateway.md (§9.7)
|
||||
- Ist-Code: backend/dialogue_turn.py (_learning_pause), backend/detect_learning.py (decision_for_mention, apply_gliner_if_enabled, auto_resolve_ambiguous)
|
||||
|
||||
Danach mit mir im Planmodus §6 der Work-Order Punkt für Punkt schließen. Nicht raten. Nicht „Bevorzugte Richtung“ als Entschieden behandeln. Erst wenn §6 schriftlich entschieden ist, einen knappen Umsetzungsplan vorschlagen (erster Schnitt vs. später). Umsetzung erst nach meiner Freigabe, dann Agent-Modus.
|
||||
|
||||
Invarianten, die nicht zur Disposition stehen:
|
||||
- Identität lokal; kein stilles Identity aus OpenRouter/GLiNER
|
||||
- keine Wortliste; Sinne/Cues aus Bestätigungen
|
||||
- Cue gilt pro Vorkommen, nicht lexemweit (Sushi-Tag-21)
|
||||
- Stufe-1-Lehrer sieht den unveränderten Nutzersatz
|
||||
- GLiNER nicht als alleiniges Detect in die Detect-URL
|
||||
- Gateway nicht abschaltbar
|
||||
- bestehenden GLiNER-Client/Cues/Review-UI umverdrahten, nicht löschen
|
||||
- Docs additiv; kanonische Kapitel nicht kürzen
|
||||
|
||||
Zielbild in einem Satz: Schülerpfad lokal zuerst, Lehrer nur im Lernmodus, zwei Stufen, Mensch bei Diff und neuer Identität.
|
||||
```
|
||||
|
|
@ -695,6 +695,16 @@ pre.code {
|
|||
flex-wrap: wrap;
|
||||
gap: 0.8rem;
|
||||
}
|
||||
fieldset.stack {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 12px;
|
||||
margin: 0.6rem 0;
|
||||
padding: 0.7rem 0.8rem 0.9rem;
|
||||
}
|
||||
fieldset.stack legend {
|
||||
padding: 0 0.3rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
.mask-excerpt {
|
||||
margin: 0.35rem 0 0;
|
||||
color: var(--ink);
|
||||
|
|
|
|||
|
|
@ -23,18 +23,35 @@ function HighlightedExcerpt({ excerpt, highlightStart, highlightEnd, label }) {
|
|||
)
|
||||
}
|
||||
|
||||
function reviewCopy(kind) {
|
||||
if (kind === 'confirm_remote') {
|
||||
return {
|
||||
title: 'Klartext-Detect bestätigen',
|
||||
status: 'GLiNER fehlt. Ein öffentliches Detect darf den Klartext nur senden, wenn du das ausdrücklich bestätigst. „Identität“ heißt senden, „Nicht schützenswert“ bricht ab.'
|
||||
}
|
||||
}
|
||||
if (kind === 'manual') {
|
||||
return {
|
||||
title: 'Maskierung prüfen',
|
||||
status: 'Manuelles Lernen: bitte jede vorgeschlagene Nennung bestätigen oder als nicht schützenswert markieren.'
|
||||
}
|
||||
}
|
||||
return {
|
||||
title: 'Maskierung prüfen',
|
||||
status: 'Abweichung oder neue Identität. Nur die gelb markierte Nennung. Bestätigen legt die Bezeichnung lokal ab. „Nicht schützenswert“ verhindert die Maskierung.'
|
||||
}
|
||||
}
|
||||
|
||||
export default function MaskReviewPanel({ review, busy, onSubmit }) {
|
||||
if (!review?.candidates?.length) return null
|
||||
const copy = reviewCopy(review.kind)
|
||||
return (
|
||||
<div className="runlog-overlay" role="dialog" aria-labelledby="mask-review-title">
|
||||
<div className="runlog-modal">
|
||||
<div className="runlog-head">
|
||||
<div>
|
||||
<h2 id="mask-review-title">Maskierung prüfen</h2>
|
||||
<p className="runlog-status">
|
||||
Lernmodus: nur die gelb markierte Nennung. Bestätigen legt die Bezeichnung lokal ab.
|
||||
„Nicht schützenswert“ verhindert die Maskierung. Beides bei demselben Wort markiert es als mehrdeutig.
|
||||
</p>
|
||||
<h2 id="mask-review-title">{copy.title}</h2>
|
||||
<p className="runlog-status">{copy.status}</p>
|
||||
</div>
|
||||
</div>
|
||||
<form
|
||||
|
|
|
|||
|
|
@ -163,9 +163,35 @@ export default function AdminProvidersPage() {
|
|||
body: { mode }
|
||||
})
|
||||
applyPayload(payload)
|
||||
setNotice(mode === 'learning'
|
||||
? 'Lernmodus: Detect pausiert vor Generate, bis Maskierungen bestätigt sind.'
|
||||
: 'Detect-Modus: semantisch, ohne Bestätigungs-Popup.')
|
||||
const labels = {
|
||||
student: 'Zielpfad: Schüler allein, ohne Lehrer-Kontrollaufruf.',
|
||||
learning_full: 'Lernen Stufe 1: Schüler und Lehrer (roher Nutzersatz) parallel.',
|
||||
learning_reduced: 'Lernen Stufe 2: Lehrer sieht keine cue-sicheren Formen mehr.',
|
||||
learning_manual: 'Manuelles Lernen: alle Schülerfunde zur Bestätigung.',
|
||||
bypass: 'Bypass: alter OpenRouter-Detect auf dem gerenderten Prompt.',
|
||||
learning: 'Lernen Stufe 1.',
|
||||
semantic: 'Bypass (alter Pfad).'
|
||||
}
|
||||
setNotice(labels[mode] || 'Detect-Modus gespeichert.')
|
||||
} catch (err) {
|
||||
setError(err.message)
|
||||
} finally {
|
||||
setSaving('')
|
||||
}
|
||||
}
|
||||
|
||||
const saveLocalNer = async (patch) => {
|
||||
setError('')
|
||||
setNotice('')
|
||||
setSaving('local-ner')
|
||||
try {
|
||||
const payload = await api('/api/admin/providers/local-ner', {
|
||||
token: session.token,
|
||||
method: 'PUT',
|
||||
body: patch
|
||||
})
|
||||
applyPayload(payload)
|
||||
setNotice('Lokale NER gespeichert. Maskierung (OpenRouter) bleibt unverändert.')
|
||||
} catch (err) {
|
||||
setError(err.message)
|
||||
} finally {
|
||||
|
|
@ -280,21 +306,72 @@ export default function AdminProvidersPage() {
|
|||
<label>
|
||||
Detect-Modus
|
||||
<select
|
||||
value={data.detect_operating_mode || 'semantic'}
|
||||
value={data.detect_operating_mode || 'learning_full'}
|
||||
onChange={(e) => saveDetectMode(e.target.value)}
|
||||
disabled={saving === 'detect-mode'}
|
||||
>
|
||||
<option value="semantic">Semantisch (Default)</option>
|
||||
<option value="learning">Lernmodus (Übergang)</option>
|
||||
<option value="student">Zielpfad (Schüler allein)</option>
|
||||
<option value="learning_full">Lernen automatisch Stufe 1</option>
|
||||
<option value="learning_reduced">Lernen automatisch Stufe 2</option>
|
||||
<option value="learning_manual">Lernen manuell</option>
|
||||
<option value="bypass">Bypass (alter OpenRouter-Pfad)</option>
|
||||
</select>
|
||||
</label>
|
||||
)}
|
||||
{role.role === 'detect' && data.detect_operating_mode === 'learning' && (
|
||||
{role.role === 'detect' && data.detect_operating_mode !== 'bypass' && (
|
||||
<p className="muted">
|
||||
Keine separate Wortliste. Bestätigungen im Dialog legen lokale Sinne an.
|
||||
Mehrdeutige Nennungen gehen nur als Mini-Passage an ein lokales Detect-Modell.
|
||||
Keine separate Wortliste. Sinne entstehen aus Bestätigungen.
|
||||
GLiNER sieht den unveränderten Nutzersatz. Der Lehrer nur im Lernmodus.
|
||||
Mini-Passagen gehen an das unten gewählte Profil, nicht an die Maskierungs-Stufe.
|
||||
</p>
|
||||
)}
|
||||
{role.role === 'detect' && (
|
||||
<fieldset className="stack">
|
||||
<legend>Lokale NER (GLiNER)</legend>
|
||||
<p className="muted">
|
||||
Unabhängig von der Maskierungs-URL. OpenRouter-Detect bleibt die Egress-Maskierung.
|
||||
</p>
|
||||
<label className="check">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={Boolean(data.local_ner?.enabled)}
|
||||
onChange={(e) => saveLocalNer({ enabled: e.target.checked })}
|
||||
disabled={saving === 'local-ner'}
|
||||
/>
|
||||
GLiNER im Schülerpfad nutzen
|
||||
</label>
|
||||
<label>
|
||||
GLiNER-URL
|
||||
<input
|
||||
defaultValue={data.local_ner?.url || 'http://192.168.2.144:8090'}
|
||||
key={data.local_ner?.url || 'gliner-url'}
|
||||
onBlur={(e) => {
|
||||
const next = e.target.value.trim()
|
||||
if (next && next !== data.local_ner?.url) saveLocalNer({ url: next })
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<p className="muted">
|
||||
Status: {data.local_ner?.health?.status || (data.local_ner?.enabled ? 'unbekannt' : 'aus')}
|
||||
{data.local_ner?.health?.model ? ` · ${data.local_ner.health.model}` : ''}
|
||||
</p>
|
||||
<label>
|
||||
Unsichere Nennung klären über
|
||||
<select
|
||||
value={data.local_ner?.passage_profile_id || ''}
|
||||
onChange={(e) => saveLocalNer({ passage_profile_id: e.target.value })}
|
||||
disabled={saving === 'local-ner'}
|
||||
>
|
||||
<option value="">Nicht klären (Popup)</option>
|
||||
{profiles.map((profile) => (
|
||||
<option key={profile.id} value={profile.id}>
|
||||
{profile.title} ({profile.local ? 'lokal' : 'extern'}{profile.model ? ` · ${profile.model}` : ''})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
</fieldset>
|
||||
)}
|
||||
<dl className="meta">
|
||||
<dt>Status</dt>
|
||||
<dd>{statusLabel(view)}</dd>
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ export default function JournalDayPage() {
|
|||
const [runLog, setRunLog] = useState([])
|
||||
const [debugOn, setDebugOn] = useState(false)
|
||||
const [maskReview, setMaskReview] = useState(null)
|
||||
const [maskReviewSource, setMaskReviewSource] = useState('turn')
|
||||
|
||||
const loadDay = async (preferId) => {
|
||||
const data = await api(`/api/journal/days/${dayId}`, { token: session.token })
|
||||
|
|
@ -71,6 +72,7 @@ export default function JournalDayPage() {
|
|||
setActiveId(nextId)
|
||||
setMessages(conv.messages || [])
|
||||
setMaskReview(conv.pending_mask_review || null)
|
||||
setMaskReviewSource(conv.pending_mask_review?.purpose === 'journal_generate' ? 'generate' : 'turn')
|
||||
} else {
|
||||
setActiveId(null)
|
||||
setMessages([])
|
||||
|
|
@ -154,6 +156,7 @@ export default function JournalDayPage() {
|
|||
setMessages(result.messages || [])
|
||||
if (result.pending_mask_review) {
|
||||
setMaskReview(result.pending_mask_review)
|
||||
setMaskReviewSource('turn')
|
||||
} else {
|
||||
setMaskReview(null)
|
||||
await loadDay(activeId)
|
||||
|
|
@ -167,10 +170,32 @@ export default function JournalDayPage() {
|
|||
}
|
||||
|
||||
const submitMaskReview = async (decisions) => {
|
||||
if (!activeId || !maskReview?.id) return
|
||||
if (!maskReview?.id) return
|
||||
setError('')
|
||||
setBusy(true)
|
||||
try {
|
||||
if (maskReviewSource === 'generate') {
|
||||
const result = await api(`/api/journal/days/${dayId}/generate/review`, {
|
||||
token: session.token,
|
||||
method: 'POST',
|
||||
body: { review_id: maskReview.id, decisions }
|
||||
})
|
||||
if (result.pending_mask_review) {
|
||||
setMaskReview(result.pending_mask_review)
|
||||
setMaskReviewSource('generate')
|
||||
return
|
||||
}
|
||||
setMaskReview(null)
|
||||
const log = result.run_log || result.trace?.log || []
|
||||
setRunLog(log)
|
||||
setTrace(result.trace || null)
|
||||
await loadDay(activeId)
|
||||
navigate(`/journal/${spaceId}/${dayId}/entry?draft=1`, {
|
||||
state: { trace: result.trace || null, run_log: log }
|
||||
})
|
||||
return
|
||||
}
|
||||
if (!activeId) return
|
||||
const result = await api(`/api/journal/conversations/${activeId}/turn/review`, {
|
||||
token: session.token,
|
||||
method: 'POST',
|
||||
|
|
@ -205,6 +230,12 @@ export default function JournalDayPage() {
|
|||
method: 'POST',
|
||||
body
|
||||
})
|
||||
if (result.pending_mask_review) {
|
||||
setMaskReview(result.pending_mask_review)
|
||||
setMaskReviewSource('generate')
|
||||
setLogOpen(false)
|
||||
return
|
||||
}
|
||||
const log = result.run_log || result.trace?.log || []
|
||||
setDecision(null)
|
||||
setTrace(result.trace || null)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user