{ "schemaVersion": 3, "publishedDate": "24 Oct 2026", "runs": [ { "id": "engineering", "model": "GPT-6.1 Sol", "reasoning": "low", "judge": "Jev 1.13.0", "date": "24 Oct 2026", "overall": { "taskPairs": 5, "withoutShifu": 5.013, "withShifu": 7.456, "wins": 5, "ties": 0, "losses": 0 }, "categories": [ { "id": "eng01", "label": "Retry recovery", "taskPairs": 1, "withoutShifu": 3.355, "withShifu": 5.46, "dimensions": [ { "id": "expected_usefulness", "label": "Expected usefulness", "withoutShifu": 2.6, "withShifu": 4.6000000000000005 }, { "id": "constraint_coverage", "label": "Constraint coverage", "withoutShifu": 1.525, "withShifu": 3.775 }, { "id": "rationale_application", "label": "Rationale application", "withoutShifu": 2.3000000000000003, "withShifu": 5.324999999999999 }, { "id": "source_faithfulness", "label": "Source faithfulness", "withoutShifu": 6.800000000000001, "withShifu": 8 }, { "id": "implementation_readiness", "label": "Implementation readiness", "withoutShifu": 3.55, "withShifu": 5.6000000000000005 } ] }, { "id": "eng02", "label": "Identity isolation", "taskPairs": 1, "withoutShifu": 5.14, "withShifu": 6.33, "dimensions": [ { "id": "expected_usefulness", "label": "Expected usefulness", "withoutShifu": 4.699999999999999, "withShifu": 5.5 }, { "id": "constraint_coverage", "label": "Constraint coverage", "withoutShifu": 4.175, "withShifu": 5 }, { "id": "rationale_application", "label": "Rationale application", "withoutShifu": 4.65, "withShifu": 5.925000000000001 }, { "id": "source_faithfulness", "label": "Source faithfulness", "withoutShifu": 7.125, "withShifu": 8.425 }, { "id": "implementation_readiness", "label": "Implementation readiness", "withoutShifu": 5.05, "withShifu": 6.800000000000001 } ] }, { "id": "eng03", "label": "Empty-result handling", "taskPairs": 1, "withoutShifu": 6.075, "withShifu": 8.035, "dimensions": [ { "id": "expected_usefulness", "label": "Expected usefulness", "withoutShifu": 5.2749999999999995, "withShifu": 7.4750000000000005 }, { "id": "constraint_coverage", "label": "Constraint coverage", "withoutShifu": 4.625, "withShifu": 7.45 }, { "id": "rationale_application", "label": "Rationale application", "withoutShifu": 4.875, "withShifu": 7.95 }, { "id": "source_faithfulness", "label": "Source faithfulness", "withoutShifu": 8.55, "withShifu": 9.225 }, { "id": "implementation_readiness", "label": "Implementation readiness", "withoutShifu": 7.05, "withShifu": 8.075 } ] }, { "id": "eng05", "label": "Correction persistence", "taskPairs": 1, "withoutShifu": 4.835, "withShifu": 8.465, "dimensions": [ { "id": "expected_usefulness", "label": "Expected usefulness", "withoutShifu": 4.25, "withShifu": 8.225 }, { "id": "constraint_coverage", "label": "Constraint coverage", "withoutShifu": 3.55, "withShifu": 7.6 }, { "id": "rationale_application", "label": "Rationale application", "withoutShifu": 3.675, "withShifu": 8.875 }, { "id": "source_faithfulness", "label": "Source faithfulness", "withoutShifu": 7.175000000000001, "withShifu": 8.8 }, { "id": "implementation_readiness", "label": "Implementation readiness", "withoutShifu": 5.525, "withShifu": 8.825 } ] }, { "id": "eng06", "label": "Historical backfill", "taskPairs": 1, "withoutShifu": 5.66, "withShifu": 8.99, "dimensions": [ { "id": "expected_usefulness", "label": "Expected usefulness", "withoutShifu": 5.074999999999999, "withShifu": 8.175 }, { "id": "constraint_coverage", "label": "Constraint coverage", "withoutShifu": 4.525, "withShifu": 8.9 }, { "id": "rationale_application", "label": "Rationale application", "withoutShifu": 4.2, "withShifu": 9.25 }, { "id": "source_faithfulness", "label": "Source faithfulness", "withoutShifu": 7.45, "withShifu": 9.45 }, { "id": "implementation_readiness", "label": "Implementation readiness", "withoutShifu": 7.05, "withShifu": 9.175 } ] } ], "highlights": [ "eng01", "eng02", "eng03", "eng05", "eng06" ], "description": "Five engineering tasks. Same model and inputs, with and without Shifu. Engineering plan quality scored by Jev out of 10.", "metricLabel": "Engineering plan quality", "reportUrl": "/benchmarks/engineering-context", "studyNote": "Internal pilot \u00b7 One work history \u00b7 One answer per condition \u00b7 Three controls excluded.", "methodology": "Five history-dependent engineering tasks were selected from real work before answer generation. Three controls are reported separately and excluded from the headline. Both conditions used fresh isolated Codex sessions, GPT-6.1 Sol at low reasoning, identical task inputs and one answer per condition. The with-Shifu condition also received the current integration skill and read-only production context retrieval. Project-specific expected constraints and source references were held out from the answering models, except in the full-context control. Explicit records from earlier benchmark activities were filtered from retrieved context; the filter is heuristic. Jev 1.13.0 scored each answer on five anchored dimensions, normalized to 0\u201310. Each answer score is the equal-weight mean of those dimensions; the headline gives each of the five primary tasks equal weight. Condition labels were hidden from the judge. Both answers were judged using the same task-specific reference packet, retrieved evidence and integration instructions. Those materials verify assertions; expected requirements and current user instructions remain authoritative.", "limitations": [ "This internal pilot covers one work history and five selected primary tasks, with one answer per condition. It does not establish statistical significance or represent all engineers, repositories or tasks.", "Scores estimate the quality of pre-coding engineering plans. Generated implementations and tests were not executed; productivity, production safety and customer outcomes were not measured.", "The with-Shifu condition combines integration instructions and retrieval, so the result does not isolate memory alone. Production retrieval is bounded and the live corpus was not frozen.", "Reference documents can contain local, uncommitted contracts or proposals. Their presence in production memory and their deployed behavior were not independently established. A retrieval gap may reflect ingestion or selection.", "The full-context control repeats one primary scenario and is not an independent task domain. Setup gains may reflect instructions supplied by the integration skill.", "Scenario descriptions are anonymized paraphrases, not verbatim benchmark prompts. Private prompts, generated answers, source records, account identifiers and implementation details are not published. Scores are unchanged." ], "metrics": [ { "id": "expected_usefulness", "label": "Expected usefulness", "withoutShifu": 4.38, "withShifu": 6.795 }, { "id": "constraint_coverage", "label": "Constraint coverage", "withoutShifu": 3.68, "withShifu": 6.545 }, { "id": "rationale_application", "label": "Rationale application", "withoutShifu": 3.94, "withShifu": 7.465 }, { "id": "source_faithfulness", "label": "Source faithfulness", "withoutShifu": 7.420000000000001, "withShifu": 8.78 }, { "id": "implementation_readiness", "label": "Implementation readiness", "withoutShifu": 5.645, "withShifu": 7.695 } ], "tieThreshold": 0.25 } ] }