Kansho/backend/tests/test_journal_generation_policy.py

"""Named journal generation guidelines: selection, snapshot, mixed sources."""
from __future__ import annotations

import os
import re
import sys
import tempfile
from pathlib import Path
from unittest.mock import patch

ROOT = Path(__file__).resolve().parents[1]
FRONTEND = ROOT.parent / "frontend"
sys.path.insert(0, str(ROOT))

os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-guidelines-test.sqlite")
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
os.environ["KANSHO_FAKE_DETECT"] = "1"
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)

from fastapi.testclient import TestClient
from db import get_db, init_db
from engine import load_active_prompt
from identity_store import confirm_identity, remember_mapping
from journal_editorial import GENERATE_SEED_REVISION
from journal_generation_policy import (
    PURPOSE_JOURNAL,
    SELECTION_KEYS,
    SLOT_TO_ID_KEY,
    CatalogError,
    GenerationPolicyError,
    archive_guideline,
    clone_guideline,
    compile_selection,
    create_guideline,
    default_selection_ids,
    get_guideline,
    get_or_create_selection,
    list_guidelines,
    load_seed_document,
    load_selection,
    overview_payload,
    publish_guideline,
    save_selection,
    seed_generation_instructions,
    set_default_guideline,
    snapshot_summary,
    update_guideline,
    validate_selection,
)
from journal_generate import unattested_journal_content
from journal_store import current_draft
from main import app
from privacy_gateway import install_test_recorder, reset_debug
from providers import ChatResult


PREVIOUS_PROMPT_CHARS = 3468
HARD_FACTS = (
    "keine neuen Tatsachen",
    "Plan und Vollzug",
    "Verneinungen",
    "Unsicherheiten",
    "[[PERSON:01]]",
)
BANNED_CODE_PHRASES = (
    "Überarbeite den Text substanziell",
    "Erhalte sämtliche belegten Ereignisse",
    "künstlich zu literarisieren",
    "Modus prose_edit",
    "Modus notes_to_journal",
    "Korrigiere nur Rechtschreibung, Grammatik und Zeichensetzung",
)
BANNED_RUNTIME_TOKENS = (
    "transformation_strength",
    "detail_retention",
    "voice_strength",
    "narrative_shaping",
    "min_value",
    "max_value",
    "remember_generation_policy",
    "generation_policy",
)
SEED = load_seed_document()
MIXED_SOURCES_INSTRUCTION = (
    "Die Quellen können aus Fließtext, Stichpunkten und Satzfragmenten bestehen. "
    "Behandle jede Passage entsprechend ihrer Form: Redigiere vorhandenen Fließtext, "
    "verwandle Stichpunkte und Fragmente in vollständige Prosa und verbinde beides zu einem einheitlichen Eintrag. "
    "Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus."
)
REQUIRED_KEYS = {
    ("transformation", "correction"),
    ("transformation", "copyedit"),
    ("transformation", "reshape"),
    ("transformation", "substantial"),
    ("detail", "compact"),
    ("detail", "selected"),
    ("detail", "broad"),
    ("detail", "complete"),
    ("voice", "neutral"),
    ("voice", "light"),
    ("voice", "noticeable"),
    ("voice", "clear"),
    ("voice", "with_examples"),
    ("voice", "legacy_neutral"),
    ("voice", "legacy_light"),
    ("voice", "legacy_noticeable"),
    ("voice", "legacy_clear"),
    ("narrative", "chronicle"),
    ("narrative", "structured"),
    ("narrative", "weighted"),
    ("narrative", "emphasized"),
}


def has_token(text: str, token: str) -> bool:
    return re.search(rf"(?<![A-Za-z0-9_]){re.escape(token)}(?![A-Za-z0-9_])", text) is not None


def expect(ok: bool, message: str) -> None:
    if not ok:
        raise SystemExit(f"FAIL: {message}")
    print(f"OK  {message}")


def header(token: str) -> dict:
    return {"X-Auth-Token": token}


def intern_of(payload: dict) -> str:
    for stage in (payload.get("trace") or {}).get("stages") or []:
        if stage.get("purpose") == "journal_generate":
            return stage.get("intern") or ""
    return (payload.get("trace") or {}).get("intern") or ""


def sources_block(intern: str) -> str:
    parts = (intern or "").split("\nCURRENT_DAY_SOURCES\n")
    if len(parts) < 2:
        return intern or ""
    rest = parts[-1]
    for marker in ("\nEXISTING_TEXT\n", "\nAUSGABE\n"):
        if marker in rest:
            rest = rest.split(marker, 1)[0]
            break
    return rest


def seed_item(slot: str, key: str) -> dict:
    for item in (SEED.get("slots") or {}).get(slot, {}).get("variants") or []:
        if item.get("guideline_key") == key:
            return item
    raise SystemExit(f"FAIL: missing seed {slot}/{key}")


def seed_id(slot: str, key: str) -> str:
    return seed_item(slot, key)["id"]


def seed_instruction(slot: str, key: str) -> str:
    return seed_item(slot, key).get("instruction") or ""


def selection_of(**keys: str) -> dict[str, str]:
    return {SLOT_TO_ID_KEY[slot]: seed_id(slot, key) for slot, key in keys.items()}


def default_ids() -> dict[str, str]:
    return selection_of(
        transformation="substantial",
        detail="complete",
        voice="clear",
        narrative="weighted",
    )


def low_ids() -> dict[str, str]:
    return selection_of(
        transformation="correction",
        detail="compact",
        voice="neutral",
        narrative="chronicle",
    )


def high_ids() -> dict[str, str]:
    return selection_of(
        transformation="substantial",
        detail="complete",
        voice="clear",
        narrative="emphasized",
    )


def selection_meta(payload: dict) -> dict:
    return (payload.get("trace") or {}).get("generation_selection") or {}


def test_runtime_has_no_numeric_policy() -> None:
    files = [
        ROOT / "journal_generation_policy.py",
        ROOT / "journal_generate.py",
        ROOT / "routers" / "journal.py",
        ROOT / "routers" / "generation_instructions.py",
        FRONTEND / "src" / "pages" / "JournalDayPage.jsx",
        FRONTEND / "src" / "pages" / "AdminGenerationPage.jsx",
    ]
    for path in files:
        text = path.read_text(encoding="utf-8")
        for token in BANNED_RUNTIME_TOKENS:
            if token == "generation_policy" and path.name == "journal_generation_policy.py":
                continue
            if token == "generation_policy" and path.name == "generation_instructions.py":
                continue
            expect(not has_token(text, token), f"{path.name} has no {token}")
    day = (FRONTEND / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
    expect("Stilquellen" in day, "voice select can show which style sources apply")
    expect("type=\"range\"" not in day, "journal page has no extra sliders for style context")
    expect(day.count("<select") == 1, "journal page maps four slots onto one select, not extra style-context fields")
    expect("policy-slider" not in day, "journal page has no slider class")
    expect("SLOT_SELECTS" in day, "journal page names the four selects")
    expect(day.count("slot:") >= 4, "journal page has four independent slots")
    expect("<select" in day, "journal page uses select fields")
    expect("Diese Auswahl als Standard merken" in day, "journal page can remember explicitly")
    expect("if (filled) return current" in day, "settings load does not overwrite a filled selection")
    expect("Auswahl nicht mehr aktiv" in day, "stale guideline ids stay visible as invalid")
    expect("Rev. ${option.revision}" in day, "options show the guideline revision")
    expect("Faktenregeln und Datenschutz bleiben unabhängig" in day, "facts/privacy note remains")
    editor = (FRONTEND / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
    expect("generation_summary" in editor, "editor can show the generation snapshot")
    admin = (FRONTEND / "src" / "pages" / "AdminGenerationPage.jsx").read_text(encoding="utf-8")
    expect("<textarea" not in admin.split("generation-admin-editor")[0], "admin overview has no textareas")
    expect(admin.count("<textarea") == 1, "admin opens one instruction editor at a time")
    expect("Stilquellen für diese Ausprägung" in admin, "admin edits style context with the instruction")
    expect("Vorgänger" in admin, "admin shows predecessor or clone origin")
    expect("include_core" in admin, "admin names core inclusion")
    expect("Writing-Profile-Core" in admin, "admin labels core in German product terms")
    expect("backend/config/" not in admin, "admin copy does not advertise seed file paths")
    expect("Quellenmodus" not in admin, "admin has no source-mode management")
    expect("source_mode" not in admin, "admin source has no source_mode slot")
    expect("prose_edit" not in admin, "admin has no prose_edit control")
    expect("notes_to_journal" not in admin, "admin has no notes_to_journal control")
    expect("Quellenmodus" not in day, "journal page has no source-mode control")
    expect("source_mode" not in day, "journal page has no source_mode field")
    expect("editorial_mode" not in day, "journal page has no editorial_mode field")
    trace_ui = (FRONTEND / "src" / "components" / "CallTrace.jsx").read_text(encoding="utf-8")
    expect("Editorial Mode" not in trace_ui, "trace UI does not present source mode as a run decision")
    expect("source_mode" not in trace_ui, "trace UI has no source_mode label")
    expect("Chunks technisch verarbeitet" in trace_ui, "trace separates technical chunk coverage")
    expect("Semantische Identitätserkennung" in trace_ui, "trace names semantic uncertainty")
    expect("nicht garantiert" in trace_ui, "trace does not claim semantic completeness")
    expect("Stilanwendung" in trace_ui, "trace shows requested versus effective style application")
    expect("cloned_from" in trace_ui, "trace can show the predecessor id")
    expect("Gespeicherter Entwurf" in trace_ui, "trace shows the stored draft")
    expect("Technisch verarbeitete Chunks bedeuten keine vollständige" in trace_ui, "trace explains coverage is not identity certainty")


def test_seed_and_compiler() -> None:
    init_db()
    for rel in ("journal_generation_policy.py", "journal_editorial.py"):
        source = (ROOT / rel).read_text(encoding="utf-8")
        for phrase in BANNED_CODE_PHRASES:
            expect(phrase not in source, f"{rel} does not hardcode {phrase!r}")
    items = list_guidelines(PURPOSE_JOURNAL, include_instruction=True)
    got = {(item["slot"], item["guideline_key"]) for item in items}
    expect(REQUIRED_KEYS <= got, "seed creates every required guideline")
    expect("source_mode" not in (SEED.get("slots") or {}), "seed document has no source_mode slot")
    expect(sum(1 for item in items if item["slot"] == "source_mode") == 0, "source modes are not seeded")
    expect(all(item["status"] == "active" for item in items if item["is_system_seed"]), "seed rows start active")

    defaults = compile_selection(default_selection_ids())
    expect(defaults.ids["transformation"] == seed_id("transformation", "substantial"), "default transformation is substantial")
    expect(
        defaults.instructions["transformation_instructions"] == seed_instruction("transformation", "substantial"),
        "default transformation instruction matches the seed",
    )
    expect(
        defaults.instructions["detail_instructions"] == seed_instruction("detail", "complete"),
        "default detail instruction matches the seed",
    )
    expect(
        defaults.instructions["voice_instructions"] == seed_instruction("voice", "clear"),
        "default voice instruction matches the seed",
    )
    expect(
        defaults.instructions["narrative_instructions"] == seed_instruction("narrative", "weighted"),
        "default narrative instruction matches the seed",
    )
    expect(defaults.seed_revision == SEED["seed_revision"], "compiled policy names the seed revision")
    expect(defaults.style_context["include_core"] is True, "default voice includes core")
    expect(defaults.style_context["include_facet"] is True, "default voice includes facet")
    expect(defaults.style_context["include_traits"] is True, "default voice includes traits")
    expect(defaults.style_context["include_style_examples"] is False, "default voice omits style examples")
    expect("source_mode_instructions" not in defaults.instructions, "compiler has no source-mode instruction")

    free = compile_selection({**default_selection_ids(), "voice_id": seed_id("voice", "neutral")})
    expect(free.style_context == {
        "include_core": False,
        "include_facet": False,
        "include_traits": False,
        "include_style_examples": False,
    }, "profile-free voice selects no style sources")
    examples = compile_selection({**default_selection_ids(), "voice_id": seed_id("voice", "with_examples")})
    expect(all(examples.style_context.values()), "with-examples voice selects all style sources")

    low = compile_selection(low_ids())
    high = compile_selection(high_ids())
    expect(low.keys["transformation"] == "correction", "low transformation is named")
    expect(high.keys["narrative"] == "emphasized", "high narrative is named")
    expect("source_mode_instructions" not in low.instructions, "low compile has no source mode")
    expect("source_mode_instructions" not in high.instructions, "high compile has no source mode")
    expect("{{" not in "".join(low.instructions.values()), "compiled instructions contain no placeholders")

    for bad in (None, [], "75", True, {"transformation_strength": 75}, {"transformation_id": seed_id("transformation", "substantial")}):
        try:
            validate_selection(bad)
            expect(False, "incomplete or numeric selection must be rejected")
        except GenerationPolicyError as exc:
            expect(exc.code == "invalid_generation_selection", "rejection uses invalid_generation_selection")

    draft = create_guideline(
        PURPOSE_JOURNAL,
        "transformation",
        {
            "guideline_key": "drafty",
            "label": "Entwurf",
            "summary": "nicht wählbar",
            "instruction": "Nur ein Draft.",
        },
    )
    try:
        validate_selection({**default_ids(), "transformation_id": draft["id"]})
        expect(False, "drafts must not be selectable")
    except GenerationPolicyError:
        expect(True, "drafts are not selectable")

    cloned = clone_guideline(seed_id("transformation", "correction"))
    original = get_guideline(seed_id("transformation", "correction"), include_instruction=True)
    update_guideline(
        cloned["id"],
        {
            "guideline_key": original["guideline_key"],
            "label": original["label"],
            "summary": original["summary"],
            "instruction": "CHANGED CLONE TEXT",
        },
    )
    expect(
        get_guideline(original["id"], include_instruction=True)["instruction"] == original["instruction"],
        "cloning does not change the original",
    )
    try:
        update_guideline(original["id"], {"label": "still active", "instruction": original["instruction"], "guideline_key": original["guideline_key"]})
        expect(False, "published guidelines must be immutable")
    except CatalogError as exc:
        expect(exc.code == "guideline_immutable", "published guidelines are not silently overwritten")

    published_clone = publish_guideline(cloned["id"])
    archived = archive_guideline(published_clone["id"])
    expect(archived["status"] == "archived", "published clones can be archived")
    try:
        validate_selection({**default_ids(), "transformation_id": published_clone["id"]})
        expect(False, "archived guidelines must not be selectable for new runs")
    except GenerationPolicyError:
        expect(True, "archived guidelines are not selectable for new runs")

    with get_db() as conn:
        seed_generation_instructions(conn)
    expect(
        get_guideline(original["id"], include_instruction=True)["instruction"] == original["instruction"],
        "seed does not overwrite published guidelines",
    )

    with get_db() as conn:
        conn.execute(
            """
            INSERT INTO generation_guidelines (
              id, purpose, slot, guideline_key, label, summary, instruction, sort_order,
              status, revision, cloned_from, is_default, is_system_seed, seed_id, seed_revision,
              created, updated
            )
            VALUES (
              'legacy-source-mode', ?, 'source_mode', 'prose_edit', 'Fließtext', '',
              'LEGACY MODE TEXT', 0, 'active', 1, NULL, 0, 0, '', '', datetime('now'), datetime('now')
            )
            """,
            (PURPOSE_JOURNAL,),
        )
        seed_generation_instructions(conn)
        archived_legacy = conn.execute(
            "SELECT status FROM generation_guidelines WHERE id = 'legacy-source-mode'"
        ).fetchone()
    expect(archived_legacy["status"] == "archived", "persisted source modes are archived")
    expect("source_mode" not in overview_payload()["slots"], "admin overview hides archived source modes")


def test_prompt_contract() -> None:
    init_db()
    prompt = load_active_prompt("mvp.journal_generate")
    text = prompt.get("template") or ""
    expect(len(text) < PREVIOUS_PROMPT_CHARS, "new prompt is shorter than the previous default")
    print(f"OK  prompt length {len(text)} after, {PREVIOUS_PROMPT_CHARS} before")
    for key in (
        "{{transformation_instructions}}",
        "{{detail_instructions}}",
        "{{voice_instructions}}",
        "{{narrative_instructions}}",
        "{{writing_profile}}",
        "{{style_examples}}",
        "{{reconstruction}}",
        "{{existing_text}}",
    ):
        expect(key in text, f"prompt contains {key}")
    expect("{{source_mode_instructions}}" not in text, "prompt has no source-mode placeholder")
    expect("{{editorial_mode}}" not in text, "prompt has no editorial_mode placeholder")
    expect("{{editorial_instructions}}" not in text, "prompt has no editorial_instructions placeholder")
    expect(MIXED_SOURCES_INSTRUCTION in text, "prompt contains the mixed-source instruction")
    expect("STYLE_EXAMPLES dienen ausschließlich" not in text, "standing prompt has no orphaned STYLE_EXAMPLES sentence")
    for fact in HARD_FACTS:
        expect(fact in text, f"hard fact rule remains: {fact}")
    expect("ich gieng zum laden" not in text, "no synthetic prose example")
    expect("nachbarhund im garten" not in text, "no synthetic notes example")
    expect("EDITORIAL_MODE" not in text, "source mode is not a user-facing prompt heading")
    expect(GENERATE_SEED_REVISION == "2026-08-29-voice-legacy-immutable-v1", "seed revision constant tracks immutable voice catalog")
    with get_db() as conn:
        row = conn.execute(
            "SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
            ("mvp.journal_generate",),
        ).fetchone()
    expect(row["seed_revision"] == "2026-08-29-voice-legacy-immutable-v1", "untouched default is updated to immutable voice catalog")
    expect(row["template"] == row["default_template"], "untouched template matches the seed")
    expect("source_mode" not in (SEED.get("slots") or {}), "source modes stay out of the seed")


def test_unattested_alias() -> None:
    mappings = [
        {
            "local_label": "Clarissa",
            "canonical_label": "Clarissa",
            "token": "PERSON:01",
            "aliases": ["Sushi"],
            "demask_label": "Clarissa",
        }
    ]
    expect(
        unattested_journal_content(
            "Clarissa kam ins Wohnzimmer.",
            ["Heute war Sushi im Wohnzimmer."],
            mappings,
            ["PERSON:01"],
        )
        is None,
        "canonical demask of an attested alias is not unattested_identity",
    )
    expect(
        unattested_journal_content(
            "Hanna kam ins Wohnzimmer.",
            ["Heute war Sushi im Wohnzimmer."],
            mappings + [{"local_label": "Hanna", "token": "PERSON:99"}],
            ["PERSON:01"],
        )
        == "unattested_identity",
        "a different person remains unattested",
    )


def main() -> None:
    test_runtime_has_no_numeric_policy()
    test_seed_and_compiler()
    test_prompt_contract()
    test_unattested_alias()

    reset_debug()
    with TestClient(app) as client:
        setup = client.post(
            "/api/auth/setup",
            json={"email": "ada@example.test", "name": "Ada", "password": "test-pass"},
        )
        headers = header(setup.json()["token"])
        profile_id = setup.json()["profile_id"]

        settings = client.get("/api/journal/generation-settings", headers=headers)
        expect(settings.status_code == 200, f"generation-settings {settings.text}")
        body = settings.json()
        expect(body["selection"] == default_ids(), "profile defaults are the named system standards")
        expect(set(body["selection"]) == set(SELECTION_KEYS), "settings expose the four guideline ids")
        for slot, options in body["options"].items():
            expect(options, f"{slot} has active options")
            expect(all("instruction" not in item for item in options), f"{slot} options hide prompt text")
            expect(all(item["id"] != "draft" for item in options), f"{slot} options are real ids")
        stored = get_or_create_selection(profile_id)
        expect({key: stored[key] for key in SELECTION_KEYS} == default_ids(), "persisted defaults match system standards")

        with get_db() as conn:
            conn.execute(
                "INSERT INTO profiles (id, email, name, password_hash, role) VALUES (?, ?, ?, ?, 'user')",
                ("other-profile", "other@example.test", "Other", "x"),
            )
        save_selection("other-profile", high_ids())
        save_selection(profile_id, low_ids())
        expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "correction"), "profile A keeps its own selection")
        expect(load_selection("other-profile")["transformation_id"] == seed_id("transformation", "substantial"), "profile B is isolated")
        save_selection(profile_id, default_ids())

        space = client.post("/api/journal/spaces", headers=headers, json={"title": "Policy"})
        day = client.post(
            f"/api/journal/spaces/{space.json()['id']}/days",
            headers=headers,
            json={"calendar_date": "2026-08-27"},
        )
        conv = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/conversations",
            headers=headers,
            json={"title": "Tag"},
        )
        turn = client.post(
            f"/api/journal/conversations/{conv.json()['id']}/turn",
            headers=headers,
            json={"body": "Ich war am Markt. Vielleicht bleibe ich kürzer. Brot holen wollte ich noch, habe es aber nicht gemacht."},
        )
        expect(turn.status_code == 200, f"turn {turn.status_code}")

        rejected = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={"conversation_ids": [conv.json()["id"]], "generation_selection": {"transformation_id": "missing"}},
        )
        expect(rejected.status_code == 400, "unknown selection is rejected")
        expect((rejected.json().get("detail") or {}).get("code") == "invalid_generation_selection", "rejection uses the selection code")
        after_reject = client.get("/api/journal/generation-settings", headers=headers).json()
        expect(after_reject["selection"] == default_ids(), "rejected values are not stored")
        save_selection(profile_id, low_ids())
        set_default_guideline(seed_id("transformation", "substantial"))
        expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "correction"), "setting a catalog default does not rewrite a stored selection")
        save_selection(profile_id, default_ids())
        blocked_calls = {"n": 0}

        def no_provider(*_a, **_k):
            blocked_calls["n"] += 1
            raise AssertionError("invalid selection must not call a provider")

        with patch("privacy_gateway.complete_model", no_provider), patch("entity_detect.complete_chat", no_provider):
            before_provider = client.post(
                f"/api/journal/days/{day.json()['day']['id']}/generate",
                headers=headers,
                json={
                    "conversation_ids": [conv.json()["id"]],
                    "generation_selection": {**default_ids(), "transformation_id": "missing"},
                },
            )
        expect(before_provider.status_code == 400, "invalid selection still fails closed")
        expect(blocked_calls["n"] == 0, "invalid selection aborts before detect and generate")

        draft = client.post(
            "/api/admin/generation-instructions/journal_generate",
            headers=headers,
            json={
                "slot": "voice",
                "guideline_key": "hidden_draft",
                "label": "versteckt",
                "summary": "Draft",
                "instruction": "Nicht für neue Läufe.",
            },
        )
        expect(draft.status_code == 200, f"admin draft {draft.text}")
        blocked_draft = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={
                "conversation_ids": [conv.json()["id"]],
                "generation_selection": {**default_ids(), "voice_id": draft.json()["id"]},
            },
        )
        expect(blocked_draft.status_code == 400, "drafts are rejected for new runs")

        reset_debug()
        recorder = install_test_recorder()
        snapshot_run = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={
                "conversation_ids": [conv.json()["id"]],
                "generation_selection": low_ids(),
                "remember_generation_selection": False,
            },
        )
        expect(snapshot_run.status_code == 200, f"low generate {snapshot_run.text}")
        intern = intern_of(snapshot_run.json())
        meta = selection_meta(snapshot_run.json())
        expect(meta.get("source") == "request", "trace names the request snapshot")
        expect(meta.get("remembered") is False, "unremembered snapshot is marked")
        expect(meta.get("ids") == {
            "transformation": seed_id("transformation", "correction"),
            "detail": seed_id("detail", "compact"),
            "voice": seed_id("voice", "neutral"),
            "narrative": seed_id("narrative", "chronicle"),
        }, "trace keeps the used ids")
        expect(meta.get("keys", {}).get("transformation") == "correction", "trace keeps guideline keys")
        expect(meta.get("revisions", {}).get("transformation") == 1, "trace keeps revisions")
        expect("editorial_mode" not in meta, "trace has no abandoned editorial mode")
        expect("source_mode" not in meta, "trace has no abandoned source mode")
        expect("Korrigiere nur" not in str(meta), "trace does not store compiled instruction text")
        expect(
            client.get("/api/journal/generation-settings", headers=headers).json()["selection"] == default_ids(),
            "unremembered snapshot does not change profile defaults",
        )
        snapshot = snapshot_run.json().get("generation_snapshot") or {}
        expect(snapshot.get("transformation", {}).get("key") == "correction", "draft stores transformation key")
        expect(snapshot.get("prompt_slug") == "mvp.journal_generate", "draft stores prompt slug")
        expect("instruction" not in str(snapshot), "draft snapshot omits instruction text")
        expect("source_mode" not in snapshot, "draft snapshot has no source_mode")
        expect("editorial_mode" not in snapshot, "draft snapshot has no editorial_mode")
        summary = snapshot_run.json().get("generation_summary") or ""
        expect(summary.startswith("Erzeugt mit:"), "draft summary is user-readable")
        expect("Quellenmodus:" not in summary, "draft summary does not name a source mode")
        expect("{{transformation_instructions}}" not in intern, "transformation placeholder is resolved")
        expect("{{detail_instructions}}" not in intern, "detail placeholder is resolved")
        expect("{{voice_instructions}}" not in intern, "voice placeholder is resolved")
        expect("{{narrative_instructions}}" not in intern, "narrative placeholder is resolved")
        expect("{{source_mode_instructions}}" not in intern, "source-mode placeholder is absent")
        expect(MIXED_SOURCES_INSTRUCTION in intern, "mixed-source instruction reaches the prompt")
        expect(seed_instruction("transformation", "correction") in intern, "selected transformation reaches the prompt")
        expect(seed_instruction("detail", "compact") in intern, "selected detail reaches the prompt")
        expect(seed_instruction("voice", "neutral") in intern, "selected voice reaches the prompt")
        expect(seed_instruction("narrative", "chronicle") in intern, "selected narrative reaches the prompt")
        expect([item.get("purpose") for item in recorder] == ["journal_generate"], "exactly one generate call")
        expect(sum(1 for item in (snapshot_run.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "run log records one model call")
        expect(
            not any("Korrigiere nur Rechtschreibung" in str(item) for item in (snapshot_run.json().get("run_log") or [])),
            "run log omits compiled prompt instructions",
        )
        for fact in HARD_FACTS:
            expect(fact in intern, f"hard fact remains at low policy: {fact}")

        copyedit_ids = selection_of(
            transformation="copyedit",
            detail="complete",
            voice="clear",
            narrative="weighted",
        )
        reset_debug()
        copyedit_recorder = install_test_recorder()
        copyedit_run = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={
                "conversation_ids": [conv.json()["id"]],
                "generation_selection": copyedit_ids,
                "remember_generation_selection": False,
            },
        )
        expect(copyedit_run.status_code == 200, f"copyedit generate {copyedit_run.text}")
        copyedit_meta = selection_meta(copyedit_run.json())
        copyedit_intern = intern_of(copyedit_run.json())
        copyedit_snap = copyedit_run.json().get("generation_snapshot") or {}
        expect(copyedit_meta.get("ids") == {
            "transformation": seed_id("transformation", "copyedit"),
            "detail": seed_id("detail", "complete"),
            "voice": seed_id("voice", "clear"),
            "narrative": seed_id("narrative", "weighted"),
        }, "copyedit request ids reach the trace")
        expect(copyedit_meta.get("keys", {}).get("transformation") == "copyedit", "copyedit key reaches the trace")
        expect(copyedit_meta.get("revisions", {}).get("transformation") == 1, "copyedit revision reaches the trace")
        expect(copyedit_meta.get("revisions", {}).get("narrative") == 1, "weighted revision of the selected id reaches the trace")
        expect(copyedit_snap.get("transformation", {}).get("id") == seed_id("transformation", "copyedit"), "snapshot stores the selected copyedit id")
        expect(copyedit_snap.get("transformation", {}).get("revision") == 1, "snapshot stores the selected copyedit revision")
        expect(copyedit_snap.get("narrative", {}).get("id") == seed_id("narrative", "weighted"), "snapshot stores the selected weighted id")
        expect(seed_instruction("transformation", "copyedit") in copyedit_intern, "selected copyedit instruction reaches the prompt")
        expect(seed_instruction("narrative", "weighted") in copyedit_intern, "selected weighted instruction reaches the prompt")
        expect(seed_instruction("transformation", "substantial") not in copyedit_intern, "unselected substantial instruction is absent")
        expect([item.get("purpose") for item in copyedit_recorder] == ["journal_generate"], "copyedit still uses one generate call")
        preview = client.post(
            "/api/admin/generation-instructions/journal_generate/preview",
            headers=headers,
            json={"generation_selection": copyedit_ids},
        )
        expect(preview.status_code == 200, f"copyedit preview {preview.text}")
        expect(preview.json()["transformation_instructions"] == seed_instruction("transformation", "copyedit"), "preview compiles the same copyedit instruction")
        expect(preview.json().get("selection", {}).get("ids", {}).get("transformation") == seed_id("transformation", "copyedit"), "preview reports the same copyedit id")
        expect(preview.json().get("selection", {}).get("revisions", {}).get("narrative") == 1, "preview reports the same weighted revision")

        notes_day = client.post(
            f"/api/journal/spaces/{space.json()['id']}/days",
            headers=headers,
            json={"calendar_date": "2026-08-28"},
        )
        notes_conv = client.post(
            f"/api/journal/days/{notes_day.json()['day']['id']}/conversations",
            headers=headers,
            json={"title": "Notizen"},
        )
        client.post(
            f"/api/journal/conversations/{notes_conv.json()['id']}/turn",
            headers=headers,
            json={"body": "- Markt\n- Brot nicht geholt\n- vielleicht kürzer bleiben"},
        )
        notes_run = client.post(
            f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
            headers=headers,
            json={"conversation_ids": [notes_conv.json()["id"]], "generation_selection": default_ids()},
        )
        expect(notes_run.status_code == 200, f"notes generate {notes_run.text}")
        notes_intern = intern_of(notes_run.json())
        notes_snap = notes_run.json().get("generation_snapshot") or {}
        expect("source_mode" not in notes_snap, "notes snapshot has no source mode")
        expect("editorial_mode" not in (notes_run.json().get("trace") or {}), "notes trace has no editorial mode")
        expect(MIXED_SOURCES_INSTRUCTION in notes_intern, "notes use the unified mixed-source instruction")
        expect("Markt" in sources_block(notes_intern), "notes sources remain complete")

        def generate_shapes(date: str, bodies: list[str], label: str) -> None:
            shaped_day = client.post(
                f"/api/journal/spaces/{space.json()['id']}/days",
                headers=headers,
                json={"calendar_date": date},
            )
            shaped_conv = client.post(
                f"/api/journal/days/{shaped_day.json()['day']['id']}/conversations",
                headers=headers,
                json={"title": label},
            )
            for body in bodies:
                client.post(
                    f"/api/journal/conversations/{shaped_conv.json()['id']}/turn",
                    headers=headers,
                    json={"body": body},
                )
            reset_debug()
            rec = install_test_recorder()
            run = client.post(
                f"/api/journal/days/{shaped_day.json()['day']['id']}/generate",
                headers=headers,
                json={"conversation_ids": [shaped_conv.json()["id"]], "generation_selection": default_ids()},
            )
            expect(run.status_code == 200, f"{label} generate {run.text}")
            shaped_intern = intern_of(run.json())
            shaped_sources = sources_block(shaped_intern)
            shaped_snap = run.json().get("generation_snapshot") or {}
            expect(MIXED_SOURCES_INSTRUCTION in shaped_intern, f"{label} has mixed-source instruction")
            expect("{{source_mode_instructions}}" not in shaped_intern, f"{label} has no source-mode placeholder")
            expect("source_mode" not in shaped_snap, f"{label} snapshot has no source_mode")
            expect("editorial_mode" not in shaped_snap, f"{label} snapshot has no editorial_mode")
            expect("editorial_mode" not in (run.json().get("trace") or {}), f"{label} trace has no editorial_mode")
            expect([item.get("purpose") for item in rec] == ["journal_generate"], f"{label} is exactly one generate call")
            for body in bodies:
                for line in body.splitlines():
                    token = line.lstrip("- ").strip()
                    if token:
                        expect(token in shaped_sources, f"{label} keeps source {token!r}")

        generate_shapes("2026-08-11", ["Ich ging zum Markt. Es war voll. Vielleicht bleibe ich kürzer."], "prose-only")
        generate_shapes("2026-08-12", ["- Kirschen\n- Brot nicht geholt\n- später Hafen"], "notes-only")
        generate_shapes(
            "2026-08-13",
            ["Ich ging zum Markt. Es war voll.", "- Kirschen\n- später Hafen"],
            "prose-then-notes",
        )
        generate_shapes(
            "2026-08-14",
            ["- Kirschen\n- später Hafen", "Ich ging zum Markt. Es war voll."],
            "notes-then-prose",
        )
        generate_shapes(
            "2026-08-15",
            ["Ich ging zum Markt. Es war voll.\n- Kirschen\n- später Hafen"],
            "mixed-in-message",
        )

        high_run = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={
                "conversation_ids": [conv.json()["id"]],
                "generation_selection": high_ids(),
                "remember_generation_selection": True,
            },
        )
        expect(high_run.status_code == 200, f"high generate {high_run.text}")
        high_intern = intern_of(high_run.json())
        expect("Überarbeite den Text substanziell" in high_intern, "high transformation reaches the prompt")
        expect("Korrigiere nur Rechtschreibung" not in high_intern, "low and high prompt blocks differ")
        expect(intern != high_intern, "low and high snapshots render different prompt blocks")
        expect(selection_meta(high_run.json()).get("remembered") is True, "remembered snapshot is marked")
        remembered = client.get("/api/journal/generation-settings", headers=headers).json()
        expect(remembered["selection"] == high_ids(), "remembered snapshot becomes the profile default")
        for fact in HARD_FACTS:
            expect(fact in high_intern, f"hard fact remains at high policy: {fact}")

        profile_run = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={"conversation_ids": [conv.json()["id"]]},
        )
        expect(profile_run.status_code == 200, f"profile generate {profile_run.text}")
        expect(selection_meta(profile_run.json()).get("source") == "profile", "omitted snapshot uses stored values")
        expect(seed_instruction("narrative", "emphasized") in intern_of(profile_run.json()), "stored high narrative is reused")

        confirm_identity(profile_id, "Clarissa", aliases=["Sushi"])
        alias_day = client.post(
            f"/api/journal/spaces/{space.json()['id']}/days",
            headers=headers,
            json={"calendar_date": "2026-08-21"},
        )
        alias_conv = client.post(
            f"/api/journal/days/{alias_day.json()['day']['id']}/conversations",
            headers=headers,
            json={"title": "Alias"},
        )
        client.post(
            f"/api/journal/conversations/{alias_conv.json()['id']}/turn",
            headers=headers,
            json={"body": "Sushi kam ins Wohnzimmer."},
        )

        def canonical_reply(_messages, _policy):
            return ChatResult(
                content="Wohnzimmer\n\nClarissa kam ins Wohnzimmer.",
                model="fake",
                usage={},
                context_compression="disabled",
            )

        with patch("privacy_gateway.complete_model", canonical_reply):
            alias_out = client.post(
                f"/api/journal/days/{alias_day.json()['day']['id']}/generate",
                headers=headers,
                json={"conversation_ids": [alias_conv.json()["id"]]},
            )
        expect(alias_out.status_code == 200, f"confirmed alias generate {alias_out.text}")
        expect("Clarissa" in (alias_out.json().get("body") or ""), "canonical spelling is kept after demask")
        expect((alias_out.json().get("trace") or {}).get("model_text_accepted") is True, "alias demask is not rejected as unattested")

        remember_mapping(profile_id, "Hanna", "PERSON:99")

        def invent(_messages, _policy):
            return ChatResult(
                content="Hanna am Hafen\n\nIch war am Markt.",
                model="fake",
                usage={},
                context_compression="disabled",
            )

        with patch("privacy_gateway.complete_model", invent):
            blocked = client.post(
                f"/api/journal/days/{day.json()['day']['id']}/generate",
                headers=headers,
                json={"conversation_ids": [conv.json()["id"]]},
            )
        expect(blocked.status_code == 409, "unattested person is still rejected")
        expect((blocked.json().get("detail") or {}).get("code") == "journal_generation_not_accepted", "409 provenance rule remains")

        created_user = client.post(
            "/api/users",
            headers=headers,
            json={"email": "policy-user@example.test", "name": "User", "password": "user-pass", "role": "user"},
        )
        expect(created_user.status_code == 200, f"create user {created_user.text}")
        user_login = client.post(
            "/api/auth/login",
            json={"email": "policy-user@example.test", "password": "user-pass"},
        )
        user_headers = header(user_login.json()["token"])
        denied = client.get("/api/admin/generation-instructions/journal_generate", headers=user_headers)
        expect(denied.status_code == 403, "admin catalog is protected")
        denied_put = client.put(
            "/api/admin/generation-instructions/journal_generate/" + seed_id("transformation", "substantial"),
            headers=user_headers,
            json={"label": "nein"},
        )
        expect(denied_put.status_code == 403, "admin catalog write is protected")

        catalog = client.get("/api/admin/generation-instructions/journal_generate", headers=headers)
        expect(catalog.status_code == 200, f"admin catalog {catalog.text}")
        expect("source_mode" not in catalog.json()["slots"], "admin catalog has no source-mode slot")
        expect("instruction" not in str(catalog.json()["slots"]["transformation"]), "admin overview omits prompt bodies")
        cloned = client.post(
            f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}/clone",
            headers=headers,
        )
        expect(cloned.status_code == 200, f"clone {cloned.text}")
        saved = client.put(
            f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}",
            headers=headers,
            json={
                "guideline_key": "substantial",
                "label": "substanziell",
                "summary": "Klon",
                "instruction": "ADMIN_CUSTOM_TRANSFORMATION_BLOCK",
            },
        )
        expect(saved.status_code == 200, f"admin draft save {saved.text}")
        published = client.post(
            f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}/publish",
            headers=headers,
        )
        expect(published.status_code == 200, f"publish {published.text}")
        expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "substantial"), "publishing a clone does not rewrite the stored selection")
        immutable = client.put(
            f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}",
            headers=headers,
            json={"label": "überschreiben", "instruction": "should not stick", "guideline_key": "substantial"},
        )
        expect(immutable.status_code == 409, "active guidelines cannot be overwritten")
        preview = client.post(
            "/api/admin/generation-instructions/journal_generate/preview",
            headers=headers,
            json={
                "generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
            },
        )
        expect(preview.status_code == 200, f"admin preview {preview.text}")
        expect(preview.json()["transformation_instructions"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK", "preview compiles locally")
        expect("(nicht enthalten)" in (preview.json().get("rendered") or ""), "preview has no personal sources")
        expect("{{transformation_instructions}}" not in (preview.json().get("rendered") or ""), "preview resolves placeholders")

        custom_run = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={
                "conversation_ids": [conv.json()["id"]],
                "generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
                "remember_generation_selection": False,
            },
        )
        expect(custom_run.status_code == 200, f"custom generate {custom_run.text}")
        expect("ADMIN_CUSTOM_TRANSFORMATION_BLOCK" in intern_of(custom_run.json()), "admin change reaches the next rendered prompt")

        init_db()
        after_seed = overview_payload()
        substantial = next(
            item for item in after_seed["slots"]["transformation"] if item["id"] == seed_id("transformation", "substantial")
        )
        expect(get_guideline(substantial["id"], include_instruction=True)["instruction"] == seed_instruction("transformation", "substantial"), "seed does not overwrite the original")
        expect(get_guideline(cloned.json()["id"], include_instruction=True)["instruction"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK", "seed does not overwrite admin clones")

        restored = client.post(
            "/api/admin/generation-instructions/journal_generate/reset",
            headers=headers,
        )
        expect(restored.status_code == 200, f"reset {restored.text}")
        drafts = [item for item in restored.json()["slots"]["transformation"] if item["status"] == "draft"]
        expect(drafts, "reset adds seed drafts")
        expect(
            get_guideline(cloned.json()["id"], include_instruction=True)["instruction"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK",
            "reset does not overwrite historical variants",
        )

        archived = client.post(
            f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}/archive",
            headers=headers,
        )
        expect(archived.status_code == 200, f"archive {archived.text}")
        archived_run = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={
                "conversation_ids": [conv.json()["id"]],
                "generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
            },
        )
        expect(archived_run.status_code == 400, "archived guidelines are rejected for new runs")

        fresh_day = client.post(
            f"/api/journal/spaces/{space.json()['id']}/days",
            headers=headers,
            json={"calendar_date": "2026-08-16"},
        )
        fresh_conv = client.post(
            f"/api/journal/days/{fresh_day.json()['day']['id']}/conversations",
            headers=headers,
            json={"title": "Legacy"},
        )
        client.post(
            f"/api/journal/conversations/{fresh_conv.json()['id']}/turn",
            headers=headers,
            json={"body": "Ich war am Markt."},
        )
        with get_db() as conn:
            current_template = conn.execute(
                "SELECT template FROM ai_prompts WHERE slug = ?",
                ("mvp.journal_generate",),
            ).fetchone()["template"]
            conn.execute(
                "UPDATE ai_prompts SET template = ? WHERE slug = ?",
                ((current_template or "") + "\n{{source_mode_instructions}}\n", "mvp.journal_generate"),
            )
        reset_debug()
        legacy_recorder = install_test_recorder()
        legacy = client.post(
            f"/api/journal/days/{fresh_day.json()['day']['id']}/generate",
            headers=headers,
            json={"conversation_ids": [fresh_conv.json()["id"]], "generation_selection": default_ids()},
        )
        expect(legacy.status_code == 409, "legacy custom prompt is refused before the provider")
        legacy_detail = legacy.json().get("detail") or {}
        expect(legacy_detail.get("code") == "prompt_contract_incompatible", "legacy prompt uses prompt_contract_incompatible")
        expect("veralteten Platzhalter" in (legacy_detail.get("message") or ""), "admin hint names the retired placeholder")
        expect(legacy_recorder == [], "legacy custom prompt does not call the provider")
        expect(current_draft(profile_id, fresh_day.json()["day"]["id"]) is None, "legacy custom prompt inserts no raw draft")
        with get_db() as conn:
            conn.execute(
                "UPDATE ai_prompts SET template = default_template WHERE slug = ?",
                ("mvp.journal_generate",),
            )

        with get_db() as conn:
            conn.execute(
                """
                UPDATE generation_guidelines
                SET instruction = ''
                WHERE id = ?
                """,
                (seed_id("transformation", "substantial"),),
            )
        reset_debug()
        broken_recorder = install_test_recorder()
        broken = client.post(
            f"/api/journal/days/{day.json()['day']['id']}/generate",
            headers=headers,
            json={"conversation_ids": [conv.json()["id"]], "generation_selection": default_ids()},
        )
        expect(broken.status_code == 409, "invalid catalog refuses generate")
        expect((broken.json().get("detail") or {}).get("code") == "generation_policy_invalid", "invalid catalog uses generation_policy_invalid")
        expect(broken_recorder == [], "invalid catalog does not call the provider")
        with get_db() as conn:
            conn.execute(
                """
                UPDATE generation_guidelines
                SET instruction = ?
                WHERE id = ?
                """,
                (seed_instruction("transformation", "substantial"), seed_id("transformation", "substantial")),
            )

        with get_db() as conn:
            conn.execute(
                "UPDATE ai_prompts SET template = 'CUSTOM POLICY PROMPT {{reconstruction}}' WHERE slug = ?",
                ("mvp.journal_generate",),
            )
        init_db()
        custom = load_active_prompt("mvp.journal_generate")
        expect(custom["template"] == "CUSTOM POLICY PROMPT {{reconstruction}}", "independently edited prompt is not overwritten")
        with get_db() as conn:
            row = conn.execute(
                "SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
                ("mvp.journal_generate",),
            ).fetchone()
        expect("{{transformation_instructions}}" in (row["default_template"] or ""), "default template still tracks mixed sources")
        expect(MIXED_SOURCES_INSTRUCTION in (row["default_template"] or ""), "default template has mixed-source instruction")
        expect(row["seed_revision"] == "2026-08-29-voice-legacy-immutable-v1", "revision updates even when template is custom")

        readable = snapshot_summary(
            {
                "transformation": {"label": "spürbar"},
                "detail": {"label": "weitgehend"},
                "voice": {"label": "deutlich"},
                "narrative": {"label": "gewichtet"},
            }
        )
        expect("Erzeugt mit:\nSpürbar · Weitgehend · Deutlich · Gewichtet" in readable, "snapshot summary is readable")
        expect("Quellenmodus" not in readable, "snapshot summary does not name a source mode")

    print("journal generation policy tests passed.")


if __name__ == "__main__":
    main()