{
  "schemaVersion": 1,
  "date": "24 Oct 2026",
  "model": "GPT-6.1 Sol",
  "reasoning": "low",
  "judge": "Jev 1.13.0",
  "overall": {
    "withoutShifu": 5.013,
    "withShifu": 7.456,
    "delta": 2.4430000000000005
  },
  "dimensions": [
    {
      "id": "expected_usefulness",
      "label": "Expected usefulness",
      "withoutShifu": 4.38,
      "withShifu": 6.795
    },
    {
      "id": "constraint_coverage",
      "label": "Constraint coverage",
      "withoutShifu": 3.68,
      "withShifu": 6.545
    },
    {
      "id": "rationale_application",
      "label": "Rationale application",
      "withoutShifu": 3.94,
      "withShifu": 7.465
    },
    {
      "id": "source_faithfulness",
      "label": "Source faithfulness",
      "withoutShifu": 7.420000000000001,
      "withShifu": 8.78
    },
    {
      "id": "implementation_readiness",
      "label": "Implementation readiness",
      "withoutShifu": 5.645,
      "withShifu": 7.695
    }
  ],
  "cases": [
    {
      "id": "eng01",
      "label": "Retry recovery",
      "description": "Plan safe recovery after an accepted upload loses its response and the client restarts, including compatibility with an older service.",
      "withoutShifu": 3.355,
      "withShifu": 5.46,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 2.6,
          "withShifu": 4.6000000000000005
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 1.525,
          "withShifu": 3.775
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 2.3000000000000003,
          "withShifu": 5.324999999999999
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 6.800000000000001,
          "withShifu": 8
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 3.55,
          "withShifu": 5.6000000000000005
        }
      ]
    },
    {
      "id": "eng02",
      "label": "Identity isolation",
      "description": "Plan a credential transition that preserves distinct private identities and handles pending work safely.",
      "withoutShifu": 5.14,
      "withShifu": 6.33,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 4.699999999999999,
          "withShifu": 5.5
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 4.175,
          "withShifu": 5
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 4.65,
          "withShifu": 5.925000000000001
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 7.125,
          "withShifu": 8.425
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 5.05,
          "withShifu": 6.800000000000001
        }
      ]
    },
    {
      "id": "eng03",
      "label": "Empty-result handling",
      "description": "Investigate why meaningful source content produces no usable evidence and the same work repeatedly becomes eligible.",
      "withoutShifu": 6.075,
      "withShifu": 8.035,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 5.2749999999999995,
          "withShifu": 7.4750000000000005
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 4.625,
          "withShifu": 7.45
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 4.875,
          "withShifu": 7.95
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 8.55,
          "withShifu": 9.225
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 7.05,
          "withShifu": 8.075
        }
      ]
    },
    {
      "id": "eng05",
      "label": "Correction persistence",
      "description": "Plan reprocessing that preserves explicit user corrections while handling concurrent edits and later updates.",
      "withoutShifu": 4.835,
      "withShifu": 8.465,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 4.25,
          "withShifu": 8.225
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 3.55,
          "withShifu": 7.6
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 3.675,
          "withShifu": 8.875
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 7.175000000000001,
          "withShifu": 8.8
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 5.525,
          "withShifu": 8.825
        }
      ]
    },
    {
      "id": "eng06",
      "label": "Historical backfill",
      "description": "Recover older missed work after an extraction upgrade without replaying the entire history.",
      "withoutShifu": 5.66,
      "withShifu": 8.99,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 5.074999999999999,
          "withShifu": 8.175
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 4.525,
          "withShifu": 8.9
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 4.2,
          "withShifu": 9.25
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 7.45,
          "withShifu": 9.45
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 7.05,
          "withShifu": 9.175
        }
      ]
    }
  ],
  "controls": [
    {
      "id": "eng04",
      "label": "Setup procedure",
      "description": "Connect a context integration while retaining manual capture settings and excluding historical imports.",
      "withoutShifu": 4.11,
      "withShifu": 7.945,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 2.9,
          "withShifu": 7.7
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 2.925,
          "withShifu": 7.4750000000000005
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 2.7,
          "withShifu": 8
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 7.7,
          "withShifu": 8.7
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 4.325,
          "withShifu": 7.8500000000000005
        }
      ]
    },
    {
      "id": "eng07",
      "label": "Current instruction priority",
      "description": "Plan onboarding under an explicit manual-only, no-schedule, no-history requirement despite older automatic defaults.",
      "withoutShifu": 6.865,
      "withShifu": 8.28,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 6.2749999999999995,
          "withShifu": 7.925
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 7,
          "withShifu": 8.75
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 5.3500000000000005,
          "withShifu": 8.875
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 8.825,
          "withShifu": 8.225
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 6.875,
          "withShifu": 7.625
        }
      ]
    },
    {
      "id": "eng08",
      "label": "Full-context control",
      "description": "Repeat the retry-recovery scenario with all critical constraints already supplied to both conditions.",
      "withoutShifu": 8.855,
      "withShifu": 9.22,
      "dimensions": [
        {
          "id": "expected_usefulness",
          "label": "Expected usefulness",
          "withoutShifu": 8.05,
          "withShifu": 9.475
        },
        {
          "id": "constraint_coverage",
          "label": "Constraint coverage",
          "withoutShifu": 9.45,
          "withShifu": 9.6
        },
        {
          "id": "rationale_application",
          "label": "Rationale application",
          "withoutShifu": 8.175,
          "withShifu": 8.9
        },
        {
          "id": "source_faithfulness",
          "label": "Source faithfulness",
          "withoutShifu": 9.65,
          "withShifu": 8.975
        },
        {
          "id": "implementation_readiness",
          "label": "Implementation readiness",
          "withoutShifu": 8.95,
          "withShifu": 9.15
        }
      ]
    }
  ],
  "methodology": [
    "Five history-dependent engineering tasks were selected from real work before answer generation. Three controls are reported separately and excluded from the headline.",
    "Both conditions used fresh isolated Codex sessions, GPT-6.1 Sol at low reasoning, identical task inputs and one answer per condition. The with-Shifu condition also received the current integration skill and read-only production context retrieval.",
    "Project-specific expected constraints and source references were held out from the answering models, except in the full-context control. Explicit records from earlier benchmark activities were filtered from retrieved context; the filter is heuristic.",
    "Jev 1.13.0 scored each answer on five anchored dimensions, normalized to 0\u201310. Each answer score is the equal-weight mean of those dimensions; the headline gives each of the five primary tasks equal weight.",
    "Condition labels were hidden from the judge. Both answers were judged using the same task-specific reference packet, retrieved evidence and integration instructions. Those materials verify assertions; expected requirements and current user instructions remain authoritative."
  ],
  "limitations": [
    "This internal pilot covers one work history and five selected primary tasks, with one answer per condition. It does not establish statistical significance or represent all engineers, repositories or tasks.",
    "Scores estimate the quality of pre-coding engineering plans. Generated implementations and tests were not executed; productivity, production safety and customer outcomes were not measured.",
    "The with-Shifu condition combines integration instructions and retrieval, so the result does not isolate memory alone. Production retrieval is bounded and the live corpus was not frozen.",
    "Reference documents can contain local, uncommitted contracts or proposals. Their presence in production memory and their deployed behavior were not independently established. A retrieval gap may reflect ingestion or selection.",
    "The full-context control repeats one primary scenario and is not an independent task domain. Setup gains may reflect instructions supplied by the integration skill.",
    "Scenario descriptions are anonymized paraphrases, not verbatim benchmark prompts. Private prompts, generated answers, source records, account identifiers and implementation details are not published. Scores are unchanged."
  ],
  "gradingAudit": {
    "initialWithoutShifu": 5.12,
    "initialWithShifu": 7.051,
    "initialDelta": 1.931,
    "note": "The first grading pass supplied the frozen contracts but omitted retrieved support for additional historical assertions. After review, every unchanged answer was regraded using the same complete verification packet for both conditions. Initial grades were retained privately; no cases were selected by their score. Score changes do not establish the cause of an individual judgment, and a single judge pass can vary."
  }
}