Kansho/backend/tests/test_journal_eval.py

"""Evaluation harness contract. Does not prove live model quality."""
from __future__ import annotations

import os
import sys
import tempfile
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from tests.harness import configure_test_engine

configure_test_engine()

os.environ["KANSHO_FAKE_PROVIDER"] = "1"
os.environ["KANSHO_FAKE_DETECT"] = "1"

from db import init_db
from journal_editorial import GENERATE_SEED_REVISION
from journal_eval import (
    FIXTURES,
    PROFILE_A,
    PROFILE_B,
    SYNTHETIC_PROSE,
    SYNTHETIC_TYPOS,
    VARIANT_BASELINE,
    VARIANT_CURRENT,
    VARIANT_PREVIOUS,
    compare_synthetic,
    fixture_context,
    score_output,
    variant_templates,
)


def expect(ok: bool, message: str) -> None:
    if not ok:
        raise SystemExit(f"FAIL: {message}")
    print(f"OK  {message}")


def main() -> None:
    init_db()
    templates = variant_templates()
    expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
    expect("Erstelle aus diesen Angaben einen ansprechenden persönlichen Tagebucheintrag" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
    expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
    expect("INHALTSTREUE" in templates[VARIANT_CURRENT], "current variant has the new contract")
    expect("{{transformation_instructions}}" in templates[VARIANT_CURRENT], "current variant compiles transformation policy")
    expect("{{source_mode_instructions}}" not in templates[VARIANT_CURRENT], "current variant has no source-mode placeholder")
    expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in templates[VARIANT_CURRENT], "current variant has mixed-source instruction")
    expect("ich gieng zum laden" not in templates[VARIANT_CURRENT], "current variant has no synthetic examples")
    expect(GENERATE_SEED_REVISION == "2026-08-29-voice-legacy-immutable-v1", "eval tracks the seeded revision constant")

    required_classes = {
        "already_narrative_with_errors",
        "bullet_points_and_fragments",
        "plan_versus_completion",
        "negation_and_uncertainty",
        "correction_of_earlier_claim",
        "imprecise_time",
        "outstanding_event_among_everyday",
        "recurring_people_and_projects",
        "incomplete_source_rebuildable",
    }
    got = {item["class"] for item in FIXTURES}
    expect(required_classes <= got, f"all required fixture classes exist, missing {required_classes - got}")
    expect(all("example.test" not in item["source"].lower() for item in FIXTURES), "fixtures stay synthetic")

    context = fixture_context(SYNTHETIC_PROSE)
    expect("source_mode_instructions" not in context, "eval context has no source-mode instruction")
    expect("editorial_mode" not in context, "eval context has no editorial mode")
    copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
    expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
    expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
    improved = score_output(
        SYNTHETIC_PROSE,
        "Ich ging dann zum Markt, auf dem es ziemlich voll war. Vielleicht bleibe ich kürzer.",
        typos=SYNTHETIC_TYPOS,
    )
    expect("zimlich" in improved["spelling_typos_fixed"], "metrics record typo fixes")
    expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
    expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
    expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")

    notes = next(item for item in FIXTURES if item["id"] == "notes_fragments")
    expect("markt" in notes["source"].lower(), "notes fixture stays notes-shaped")
    incomplete = next(item for item in FIXTURES if item["id"] == "incomplete_clause")
    expect("danach sprach ich mit dem" in incomplete["source"].lower(), "incomplete fixture stays incomplete prose")

    report = compare_synthetic(live=False)
    expect(report["live"] is False, "default eval is offline")
    expect(report["live_quality_confirmed"] is False, "offline run does not confirm live quality")
    expect(report["winner_declared"] is False, "harness does not declare a winner")
    expect(len(report["fixtures"]) == len(FIXTURES), "offline report covers every fixture")
    first = report["fixtures"][0]
    expect("human_blind" in first and "prompt_1" in first["human_blind"], "each fixture has a blind pair")
    expect(first["human_blind"]["hidden_mapping"]["prompt_1"] == VARIANT_CURRENT, "mapping stays machine-side")
    expect("Kanshō habe gewonnen" not in report["note"], "no victory claim")
    expect(any(item["variant"] == VARIANT_CURRENT and item["fake_provider"] for item in first["variants"]), "offline current variant is fake")

    ab = compare_synthetic(live=False, profile_ab=True)
    expect(ab["profile_ab"]["same_facts"] is True, "profile A/B keeps facts identical")
    expect(ab["profile_ab"]["prompts_differ"] is True, "rendered prompts contain different style briefs")
    expect(PROFILE_A[:20] != PROFILE_B[:20], "synthetic profiles are distinct")
    expect("Live-Prosa" in (ab["profile_ab"]["note"] or ""), "offline A/B does not claim live prose")
    print("journal eval tests passed.")


if __name__ == "__main__":
    main()