{
  "format": "ta-amps.seed-pack.proposal.v1",
  "status": "proposal_only",
  "pack_id": "evaluation-design",
  "title": "Evaluation design",
  "version": "0.1.0",
  "summary": "A collection of nouns and verbs for fair, pre-registered evaluations: arms, frozen sets with manifests, pass marks committed before runs, exact-match scoring, two runs per arm, cost per case and missed marks reported, with a worked example the validator recomputes.",
  "license": "MIT",
  "store": "seedpack.evaluation-design",
  "queue_placeholder": "conformance-seed-packs",
  "files": [
    {
      "path": "README.md",
      "sha256": "45aa5b9a8ad8d67f3db695780a71d02f0bea530bc24cabf4d55e1e046052d8d8"
    },
    {
      "path": "source.json",
      "sha256": "c5cf97616966fd72b6749a3e2021322a266e48656f013bbdc2b4005a9d198214"
    },
    {
      "path": "nouns.json",
      "sha256": "cb10cd41d6d9c8d74dca1b32d84361804facdf8d2546170ce2eae7a51fc1ebbf"
    },
    {
      "path": "verbs.json",
      "sha256": "5fb0a3be9ec2d890dd56d9b0dec63b2a7193bf3ab95f02e5a73d012226c6c1fc"
    }
  ],
  "nouns": [
    {
      "key": "evaluation-design.v0.1.0.pack",
      "category_id": "noun.skill_pack",
      "role": "pack",
      "value_sha256": "f8f99bc44d6c4cba00a4723de51d23edd1fe53cc2da629b78a6dacdef2fa22e3"
    },
    {
      "key": "evaluation-design.v0.1.0.method",
      "category_id": "noun.skill_pack",
      "role": "method",
      "value_sha256": "436329f710d068a71b3e6f7252a8f4ff875bda0d6eac92424526cb5712be11db"
    },
    {
      "key": "evaluation-design.v0.1.0.scoring.exact-match",
      "category_id": "noun.reflex_procedure",
      "role": "scoring_procedure",
      "value_sha256": "677024a78f60eef2c8391d61e0ca0a77e71a3443477b6bd6d64d0d400a68ae27"
    },
    {
      "key": "evaluation-design.v0.1.0.example.worked",
      "category_id": "noun.benchmark_suite",
      "role": "worked_example",
      "value_sha256": "68a2a02056104e275efaf2ea42233bf1276c33c9c9d4b18d5429fa7f1c859849"
    },
    {
      "key": "evaluation-design.v0.1.0.template.evaluation-plan",
      "category_id": "noun.benchmark_suite",
      "role": "template",
      "value_sha256": "6a0574ec2e905d769ae86efeed27fe4cb64936280ce5a0068aaf1400099d9718"
    },
    {
      "key": "evaluation-design.v0.1.0.template.arm",
      "category_id": "noun.skill_pack",
      "role": "template",
      "value_sha256": "d3a60352f0d3260cbd4b344b723a28966532a496d163460f90ed5522d56c4691"
    },
    {
      "key": "evaluation-design.v0.1.0.template.frozen-set-manifest",
      "category_id": "noun.benchmark_suite",
      "role": "template",
      "value_sha256": "1639cfc067a9995a94d5e134ef35f6e1f6053a89d123fe767c602e94a82fe77e"
    },
    {
      "key": "evaluation-design.v0.1.0.template.pass-marks",
      "category_id": "noun.seed_hypothesis",
      "role": "template",
      "value_sha256": "c5c105e33276b5eb4bcfe772512d3df5fee7e189b69799c9bc356437046076e0"
    },
    {
      "key": "evaluation-design.v0.1.0.template.run-record",
      "category_id": "noun.eval_run",
      "role": "template",
      "value_sha256": "4351a2712399c92113b08bc2f9fa75211e259332b84cd4e63479fa701ead40eb"
    },
    {
      "key": "evaluation-design.v0.1.0.template.evaluation-report",
      "category_id": "noun.eval_run",
      "role": "template",
      "value_sha256": "ac06027569cdd0cc1249ebdb3ed3b471c72bf03a08cf123027061cbf1c726096"
    }
  ],
  "verbs": [
    {
      "template_id": "evaluation-design.design-evaluation",
      "kind": "verb.problemize",
      "claimant": "agent",
      "idempotency_key": "seedpack:evaluation-design:design-evaluation:fc8645d987211134a4abd9080263804fef8f19faa0ffc10604f1fa177bad3eb3",
      "report_shape_sha256": "05cacbabf4906bbe03a524800b646e4f41f26c43d3773a1beb739a5920fa1092"
    },
    {
      "template_id": "evaluation-design.freeze-set",
      "kind": "verb.ingest",
      "claimant": "agent",
      "idempotency_key": "seedpack:evaluation-design:freeze-set:6fd5d6d81897e153a395f63977048ed9248b05e8bbdc0c3d128d5104e3f079e8",
      "report_shape_sha256": "8761953a2cd2212151135553651b2b4a29b377ed4c4ee2b7485d5807a509f731"
    },
    {
      "template_id": "evaluation-design.commit-pass-marks",
      "kind": "verb.verify_gate",
      "claimant": "human",
      "idempotency_key": "seedpack:evaluation-design:commit-pass-marks:16af60e0f070ed1db68e0555031c49e143cfd1a139754ed76ef728e7b17aa479",
      "report_shape_sha256": "99c528d769cb3a20fd2f726fa21ddf5b7f0992718c25613d7065a22acec7ac52"
    },
    {
      "template_id": "evaluation-design.run-arm",
      "kind": "verb.benchmark_harness",
      "claimant": "agent",
      "idempotency_key": "seedpack:evaluation-design:run-arm:0ac25f1db9b9894b70e7687f2f3114b76899fe620e905b94d80a300d3e259307",
      "report_shape_sha256": "1089cb573f6c6525f2329e5b4626ea4f4dbc21ff2cca225fbcb23cd2deced251"
    },
    {
      "template_id": "evaluation-design.score-run",
      "kind": "verb.verify_gate",
      "claimant": "agent",
      "idempotency_key": "seedpack:evaluation-design:score-run:7f5d96114893a9eb2a36c258201329707b8a677c947f3b063341a880c8d17a59",
      "report_shape_sha256": "ca3c62eb527015103fb7744f6ebb787432fb3a43be1ae9b04a6a45c3cb06396a"
    },
    {
      "template_id": "evaluation-design.report-evaluation",
      "kind": "verb.solve_draft",
      "claimant": "agent",
      "idempotency_key": "seedpack:evaluation-design:report-evaluation:0673f728a22bcb4bdc01a9bae0a0ca61bff1f2f10c12aed4af9e56dbdec879dd",
      "report_shape_sha256": "562561c14b0afa9ac44430873ca0f1ee3700866e2345d23ec54160e00e96c957"
    },
    {
      "template_id": "evaluation-design.review-evaluation",
      "kind": "verb.verify_gate",
      "claimant": "agent",
      "idempotency_key": "seedpack:evaluation-design:review-evaluation:cdac1bf1f6e5f61c46079855b411e20b6e3053647996090e86806548312bc436",
      "report_shape_sha256": "ebcc29470fa209f0096d5078fa3fd7576d9b5fb2087958a8464a37151d9c4164"
    }
  ],
  "dependencies": [],
  "boundaries": [
    "Pass marks are committed by a person before any scored run and never change afterwards.",
    "Labels are readable only by the scorer.",
    "Importing the pack writes nothing and enqueues nothing."
  ],
  "override_policy": {
    "automatic_merge": false,
    "on_conflict": "stop_and_clarify"
  },
  "importer": null,
  "importer_reason": "No manifest endpoint or generic importer exists. Reading or exporting this pack writes no records, enqueues no verbs and activates nothing.",
  "activation": {
    "automatic": false,
    "steps": [
      "An owner assigns the namespace, store and queue.",
      "Each noun is written create-only with existing grants and read back; its value SHA-256 must match the pin.",
      "A verb template is filled with exact pins and enqueued explicitly by an authorized requester."
    ]
  },
  "evaluation": {
    "result": null,
    "reason": "Offline shape, category and digest validation only. No worker has run these verbs."
  }
}