{
  "suite_id": "ofone-v0.5-three-arm-evaluation",
  "version": "0.1.0",
  "claim_boundary": "OfOne must not claim empirical superiority until this suite or a stricter successor has completed enough cases with released results and failure analysis.",
  "arms": [
    {
      "arm_id": "direct_answer",
      "label": "Direct answer",
      "description": "Answer the user objective directly without a required map artifact.",
      "required_outputs": ["answer", "confidence_or_uncertainty", "source_or_gap_notes"]
    },
    {
      "arm_id": "light_structured",
      "label": "Light structured prompt",
      "description": "Use a conventional lightweight structure such as bullets, SWOT, pros/cons, or checklist without typed OfOne artifacts.",
      "required_outputs": ["structured_answer", "risks_or_unknowns", "recommendation"]
    },
    {
      "arm_id": "full_ofone",
      "label": "Full OfOne",
      "description": "Produce a validated OfOne artifact plus renderer output and patch report when the case includes an update event.",
      "required_outputs": ["artifact_json", "validator_result", "rendering", "patch_report_if_update_case"]
    }
  ],
  "task_families": [
    "strategic",
    "scientific",
    "formal",
    "normative",
    "hybrid",
    "update_patch"
  ],
  "metrics": [
    {
      "metric_id": "decision_quality",
      "label": "Decision quality",
      "scale": "1-5",
      "description": "Recommendation quality, criteria fit, reversibility, and actionability."
    },
    {
      "metric_id": "evidence_grounding_precision",
      "label": "Evidence grounding precision",
      "scale": "1-5",
      "description": "Evidence/claim separation, provenance, source identity, and unsupported-claim avoidance."
    },
    {
      "metric_id": "uncertainty_calibration",
      "label": "Uncertainty calibration",
      "scale": "1-5",
      "description": "Visibility of unknowns, confidence basis, contradiction load, and hidden-variable risk."
    },
    {
      "metric_id": "trace_completeness",
      "label": "Trace completeness",
      "scale": "1-5",
      "description": "Ability to trace recommendation dependencies through claims, edges, criteria, options, gates, and rendering."
    },
    {
      "metric_id": "auditability",
      "label": "Auditability",
      "scale": "1-5",
      "description": "Reconstructability of evidence, claims, validation state, review state, and release conditions."
    },
    {
      "metric_id": "update_quality",
      "label": "Update quality",
      "scale": "1-5",
      "description": "Correctness and minimality of patch closure, trigger handling, and rendering regeneration."
    },
    {
      "metric_id": "cost",
      "label": "Cost",
      "scale": "1-5 inverted",
      "description": "Relative token, time, and review burden. Lower burden earns higher score only when quality remains adequate."
    },
    {
      "metric_id": "inter_run_stability",
      "label": "Inter-run stability",
      "scale": "1-5",
      "description": "Consistency of map structure, conclusions, gates, and patch behavior across repeated runs."
    }
  ],
  "minimums_before_superiority_claim": {
    "cases_per_family": 3,
    "total_cases": 21,
    "runs_per_case_per_arm": 3,
    "model_families": 2,
    "publish_failure_analysis": true,
    "case_distribution_note": "The suite has six declared task families. Minimum superiority evidence requires at least 3 cases per family plus enough additional cross-family cases to reach 21 total cases."
  },
  "benchmark_batches": [
    {
      "batch_id": "2026-05-17-batch-01",
      "label": "Initial five-case benchmark execution plan",
      "status": "in_progress",
      "manifest": "benchmarks/runs/2026-05-17-batch-01/manifest.json"
    }
  ],
  "cases": [
    {
      "case_id": "case-strategic-gated-diligence-001",
      "title": "Strategic gated diligence",
      "families": ["strategic", "update_patch"],
      "case_file": "benchmarks/cases/strategic-gated-diligence.md",
      "rubric": "benchmarks/rubrics/decision-map-rubric.md",
      "arms": ["direct_answer", "light_structured", "full_ofone"],
      "ofone_artifact": "examples/strategy-micro.json"
    },
    {
      "case_id": "case-regulated-wastewater-market-entry-001",
      "title": "Regulated wastewater market entry",
      "families": ["strategic", "scientific", "normative", "hybrid", "update_patch"],
      "case_file": "benchmarks/cases/regulated-wastewater-market-entry.md",
      "rubric": "benchmarks/rubrics/decision-map-rubric.md",
      "arms": ["direct_answer", "light_structured", "full_ofone"],
      "ofone_artifact": "examples/source-backed-wastewater-map.json"
    },
    {
      "case_id": "case-formal-proof-search-001",
      "title": "Formal proof search",
      "families": ["formal"],
      "case_file": "benchmarks/cases/formal-proof-search.md",
      "rubric": "benchmarks/rubrics/decision-map-rubric.md",
      "arms": ["direct_answer", "light_structured", "full_ofone"],
      "ofone_artifact": "examples/formal-proof-map.json"
    },
    {
      "case_id": "case-public-sector-ai-policy-audit-001",
      "title": "Public-sector AI policy audit",
      "families": ["normative", "hybrid", "update_patch"],
      "case_file": "benchmarks/cases/public-sector-ai-policy-audit.md",
      "rubric": "benchmarks/rubrics/decision-map-rubric.md",
      "arms": ["direct_answer", "light_structured", "full_ofone"],
      "ofone_artifact": "examples/hybrid-policy-audit.json"
    },
    {
      "case_id": "case-scientific-mechanism-check-001",
      "title": "Scientific mechanism check",
      "families": ["scientific"],
      "case_file": "benchmarks/cases/scientific-mechanism-check.md",
      "rubric": "benchmarks/rubrics/decision-map-rubric.md",
      "arms": ["direct_answer", "light_structured", "full_ofone"],
      "ofone_artifact": "examples/scientific-mechanism-map.json"
    }
  ]
}