{
  "benchmark_id": "herma-shim-lift-and-natural-routing-20261030",
  "as_of": "2026-10-30",
  "scope_note": "Synthetic pre-registered evaluation comparing raw zero-context calls, Herma zero-weight context shimming, and promoted PEFT LoRA / QLoRA adapters across post-cutoff Task Object (noun) and Task (verb) contracts, plus natural-language builtin starter routing parity across Python and Cloudflare Worker.",
  "tool_call_and_schema_benchmarks": [
    {
      "model": "Llama-3.2-3B-Instruct",
      "eval_set": "frozen-synthetic-150",
      "rows": 150,
      "base_exact_pct": 52.0,
      "herma_shim_zero_weight_pct": 64.7,
      "herma_shim_delta_pp": 12.7,
      "promoted_lora_pct": 72.7,
      "promoted_lora_delta_pp_vs_base": 20.7,
      "weight_updates_for_shim": 0
    },
    {
      "model": "Llama-3.2-1B-Instruct",
      "eval_set": "frozen-synthetic-150",
      "rows": 150,
      "base_exact_pct": 12.0,
      "promoted_lora_pct": 50.7,
      "promoted_lora_delta_pp_vs_base": 38.7
    },
    {
      "model": "Muse-Glimmer-30B-QLoRA",
      "eval_set": "fresh-preregistered-holdout-150",
      "rows": 150,
      "base_exact_pct": 93.3,
      "promoted_qlora_pct": 99.3,
      "promoted_qlora_delta_pp_vs_base": 6.0,
      "base_valid_json_pct": 96.7,
      "promoted_valid_json_pct": 100.0,
      "base_refusal_pct": 97.3,
      "promoted_refusal_pct": 99.3
    }
  ],
  "natural_language_shim_routing": {
    "runtime_parity": "100% Python <-> Cloudflare Worker",
    "contributor_tier_refusal_pct": 100.0,
    "verified_queries": [
      {
        "query": "What are tasks and task objects?",
        "matched_provision": "PROV-DIRECT-NOUN-VERB-SCHEMA",
        "action": "respond_directly",
        "upstream_llm_tokens": 0
      },
      {
        "query": "task and task-object schema",
        "matched_provision": "PROV-DIRECT-NOUN-VERB-SCHEMA",
        "action": "respond_directly",
        "upstream_llm_tokens": 0
      },
      {
        "query": "agent harness feeds",
        "matched_provision": "PROV-HARNESS-FEEDS-GUIDE",
        "action": "shim_context",
        "shimmed_refs": [
          "noun:harness.feed",
          "detail:herma.shim_budget"
        ]
      },
      {
        "query": "prompt shimming vs lora as parametric memory",
        "matched_provision": "PROV-LORA-MEMORY-ROUTING",
        "action": "shim_context",
        "shimmed_refs": [
          "detail:herma.lora_vs_shim",
          "detail:herma.shim_budget"
        ]
      }
    ]
  }
}