mitai-jinkendo/backend/migrations/020_unified_prompt_system.sql

-- Migration 020: Unified Prompt System (Issue #28)
-- Consolidate ai_prompts and pipeline_configs into single system
-- Type: 'base' (reusable building blocks) or 'pipeline' (workflows)

-- Step 1: Add new columns to ai_prompts and make template nullable
ALTER TABLE ai_prompts
ADD COLUMN IF NOT EXISTS type VARCHAR(20) DEFAULT 'pipeline',
ADD COLUMN IF NOT EXISTS stages JSONB,
ADD COLUMN IF NOT EXISTS output_format VARCHAR(10) DEFAULT 'text',
ADD COLUMN IF NOT EXISTS output_schema JSONB;

-- Make template nullable (pipeline-type prompts use stages instead)
ALTER TABLE ai_prompts
ALTER COLUMN template DROP NOT NULL;

-- Step 2: Migrate existing single-prompts to 1-stage pipeline format
-- All existing prompts become single-stage pipelines with inline source
UPDATE ai_prompts
SET
    type = 'pipeline',
    stages = jsonb_build_array(
        jsonb_build_object(
            'stage', 1,
            'prompts', jsonb_build_array(
                jsonb_build_object(
                    'source', 'inline',
                    'template', template,
                    'output_key', REPLACE(slug, 'pipeline_', ''),
                    'output_format', 'text'
                )
            )
        )
    ),
    output_format = 'text'
WHERE stages IS NULL;

-- Step 3: Migrate pipeline_configs into ai_prompts as multi-stage pipelines
-- Each pipeline_config becomes a pipeline-type prompt with multiple stages
INSERT INTO ai_prompts (
    slug,
    name,
    description,
    type,
    stages,
    output_format,
    active,
    is_system_default,
    category
)
SELECT
    'pipeline_config_' || LOWER(REPLACE(pc.name, ' ', '_')) || '_' || SUBSTRING(pc.id::TEXT FROM 1 FOR 8) as slug,
    pc.name,
    pc.description,
    'pipeline' as type,
    -- Build stages JSONB: combine stage1_prompts, stage2_prompt, stage3_prompt
    (
        -- Stage 1: Convert array to prompts
        SELECT jsonb_agg(stage_obj ORDER BY stage_num)
        FROM (
            SELECT 1 as stage_num,
                   jsonb_build_object(
                       'stage', 1,
                       'prompts', (
                           SELECT jsonb_agg(
                               jsonb_build_object(
                                   'source', 'reference',
                                   'slug', s1.slug_val,
                                   'output_key', REPLACE(s1.slug_val, 'pipeline_', 'stage1_'),
                                   'output_format', 'json'
                               )
                           )
                           FROM UNNEST(pc.stage1_prompts) AS s1(slug_val)
                       )
                   ) as stage_obj
            WHERE array_length(pc.stage1_prompts, 1) > 0

            UNION ALL

            SELECT 2 as stage_num,
                   jsonb_build_object(
                       'stage', 2,
                       'prompts', jsonb_build_array(
                           jsonb_build_object(
                               'source', 'reference',
                               'slug', pc.stage2_prompt,
                               'output_key', 'synthesis',
                               'output_format', 'text'
                           )
                       )
                   ) as stage_obj
            WHERE pc.stage2_prompt IS NOT NULL

            UNION ALL

            SELECT 3 as stage_num,
                   jsonb_build_object(
                       'stage', 3,
                       'prompts', jsonb_build_array(
                           jsonb_build_object(
                               'source', 'reference',
                               'slug', pc.stage3_prompt,
                               'output_key', 'goals',
                               'output_format', 'text'
                           )
                       )
                   ) as stage_obj
            WHERE pc.stage3_prompt IS NOT NULL
        ) all_stages
    ) as stages,
    'text' as output_format,
    pc.active,
    pc.is_default as is_system_default,
    'pipeline' as category
FROM pipeline_configs pc;

-- Step 4: Add indices for performance
CREATE INDEX IF NOT EXISTS idx_ai_prompts_type ON ai_prompts(type);
CREATE INDEX IF NOT EXISTS idx_ai_prompts_stages ON ai_prompts USING GIN (stages);

-- Step 5: Add comment explaining stages structure
COMMENT ON COLUMN ai_prompts.stages IS 'JSONB array of stages, each with prompts array. Structure: [{"stage":1,"prompts":[{"source":"reference|inline","slug":"...","template":"...","output_key":"key","output_format":"text|json"}]}]';

-- Step 6: Backup pipeline_configs before eventual deletion
CREATE TABLE IF NOT EXISTS pipeline_configs_backup_pre_020 AS
SELECT * FROM pipeline_configs;

-- Note: We keep pipeline_configs table for now during transition period
-- It can be dropped in a later migration once all code is migrated