{ "pack_id": "continuous-self-review", "title": "Continuous self-review", "version": "0.1.0", "summary": "A collection of nouns and verbs for reviewing a repository, site or video on a rotation, run after run: scopes with the checks each needs from other packs, a schedule that stays a proposal until a person approves it, the next scopes picked by a fixed rule from what was reviewed least recently or changed most, each run's findings diffed against the last run into new, still open, fixed, unverified, unexplained, unchecked and reopened, fixes verified by a different worker, a review log that shows coverage over time, and a self-review before any change is handed on.", "dependencies": [ "findings-to-fixes", "unattended-run-record", "regression-watch", "video-qa" ], "boundaries": [ "A schedule is a proposal until a person approves it; this pack activates no scheduler and enqueues nothing.", "The checks belong to their own packs and run as their own verbs; this pack picks, diffs and logs.", "The review log is append-only, updated by compare-and-set; no entry is edited or removed.", "A worker never verifies its own fix or approves its own change.", "Workers create nouns and propose verbs; an authorized requester enqueues the proposals worth doing.", "Importing the pack writes nothing and enqueues nothing." ], "nouns": [ { "slug": "pack", "category": "noun.skill_pack", "role": "pack", "title": "A collection of nouns and verbs for reviewing a repository, site or video on a rotation, run after run", "value": { "lets_workers": "Cut a repository, site or video into scopes with the checks each needs from other packs; propose a review schedule that waits for a person's approval; pick the next scopes by a fixed rule from what was reviewed least recently or changed most; diff each run's findings against the last run of the same scope, so new, still open, fixed, unverified, unexplained, unchecked and reopened findings are told apart; propose the fix, verification and reproduction each one needs, with a different worker verifying; keep a review log that shows coverage over time; and review one's own change before handing it on.", "flow": [ { "verb": { "$verb": "continuous-self-review.map-scopes" }, "when": "Once per target, and again when its structure changes." }, { "verb": { "$verb": "continuous-self-review.approve-schedule" }, "when": "A person approves the schedule, with an expiry; until then nothing runs on it." }, { "verb": { "$verb": "continuous-self-review.pick-scope" }, "when": "Each approved window, or by hand." }, { "verb": { "$verb": "continuous-self-review.diff-against-last-run" }, "when": "After the picked scope's checks report." }, { "verb": { "$verb": "continuous-self-review.log-run" }, "when": "After every diff." }, { "verb": { "$verb": "findings-to-fixes.verify-fix" }, "when": "A different worker verifies every fix the diff finds unverified." }, { "verb": { "$verb": "continuous-self-review.review-coverage" }, "when": "Monthly, across the log." }, { "verb": { "$verb": "continuous-self-review.review-own-change" }, "when": "Before a worker hands any change on." } ], "guardrails": [ { "$pin": "guardrail.schedule-is-a-proposal" }, { "$pin": "guardrail.gone-is-not-fixed" }, { "$pin": "regression-watch:guardrail.compare-like-with-like" }, { "$pin": "video-qa:guardrail.not-run-is-not-a-pass" } ], "worked_examples": [ { "$pin": "loop-checks" }, { "$pin": "example.scope-catalog" }, { "$pin": "example.review-schedule" }, { "$pin": "example.schedule-approval" }, { "$pin": "example.scope-pick" }, { "$pin": "example.review-run" }, { "$pin": "example.review-log" }, { "$pin": "example.self-review" }, { "$pin": "example.coverage-review" } ], "complements": [ "The checks themselves belong to other packs, such as codebase health review, edge and corner cases, accessibility audit, screenshot design review, video QA, annoyance hunt and wiring and conformance checks; this pack decides which scope runs when and what changed since last time.", "Findings, fixes, independent verification and guards come from the findings to fixes pack; a cause that keeps coming back goes to its review-patterns.", "The person's approval reuses the unattended run record pack's run approval, and an unattended window follows its run guards.", "Comparing measured numbers with an accepted baseline is the regression watch pack; this pack compares findings with the last run." ] }, "relations": [ [ "requires_noun", "method" ], [ "requires_noun", "guardrail.schedule-is-a-proposal" ], [ "requires_noun", "guardrail.gone-is-not-fixed" ], [ "requires_noun", "rotation-rule" ], [ "requires_noun", "diff-rules" ], [ "requires_noun", "scope-kinds" ], [ "requires_noun", "self-review-rules" ], [ "requires_noun", "loop-checks" ] ] }, { "slug": "method", "category": "noun.skill_pack", "role": "method", "title": "How a repository, site or video is reviewed on a rotation, run after run", "category_reason": "A reusable procedure with guardrails, followed by every verb in this pack.", "value": { "lifecycle": [ "map-scopes cuts the target into scopes, names each scope's checks from other packs, and proposes a review schedule.", "approve-schedule is a person's verb: until it is recorded, the schedule is only a proposal and nothing runs on it.", "pick-scope applies the rotation rule to the review log and proposes the picked scopes' checks, at most ten verbs per pick.", "The checks run as their own packs' verbs, each with its own receipt.", "diff-against-last-run compares this run's findings with the last run of the same scope and sorts each into new, still open, fixed, gone without verification, gone unexplained, not checked or reopened.", "log-run appends the run to the review log by compare-and-set, so coverage over time stays visible.", "review-coverage reads the log across a window for scopes left out, checks that never fail or never run, and causes that keep coming back.", "review-own-change is the self-review a worker runs on its own change before handing it to a different worker." ], "rules": [ "A schedule is a proposal until a person approves it, with scopes, an allowance and an expiry.", "Gone is not fixed: a finding is fixed only when a fix is recorded and a worker other than the fixer verified it.", "A check that did not run is not a pass; the findings it would have seen stay open as not checked.", "Diff only like with like: the same scope and the same check versions as the last run.", "A finding seen again after it was closed is reopened, and its guard failed to catch it, so a stronger guard is proposed.", "Self-review comes first and is never the last word: the worker that made a change never approves or verifies it.", "Workers create nouns and propose verbs; they never enqueue." ] } }, { "slug": "guardrail.schedule-is-a-proposal", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: a review schedule is a proposal until a person approves it", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "A review schedule is a proposal until a person approves it. The approval names the scopes, the cadence, the allowance per window and an expiry; nothing runs on a schedule whose approval is missing or expired, and changing the schedule needs a new approval.", "how_it_is_checked": "Every scheduled run names an approval that covers its scope and has not expired; a run without one says manual and names who asked for it." } }, { "slug": "guardrail.gone-is-not-fixed", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: a finding that is not seen again is not yet fixed", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "A finding that the last run saw and this run did not is fixed only when a fix is recorded and a worker other than the fixer verified it. Without a fix it is gone unexplained and is reproduced; with an unverified fix it waits for verification; if its check did not run, it stays open.", "how_it_is_checked": "Every fixed row in a diff names its fix and a verification by a different worker or a person." } }, { "slug": "rotation-rule", "category": "noun.reflex_procedure", "role": "rotation_rule", "title": "Picking the next scopes: least recently reviewed or changed most", "category_reason": "A zero-call rule: the same log, catalog and churn always pick the same scopes.", "value": { "order": [ "Scopes never reviewed come first, in catalog order.", "Urgent scopes come next: those more than twice their max interval since their last review, most overdue first by days over interval.", "Every other scope gets score = days since its last review / its max interval + min(churn since then / its churn cap, 1) + 0.5 x min(open P0 and P1 findings, 2), rounded to three places, highest first.", "Ties go to the lower scope id." ], "pick": "Take the first scopes_per_window scopes the approved schedule allows.", "overdue": "A scope is overdue when its days since review exceed its max interval, and urgent when they exceed twice it.", "no_starving": "However busy other scopes are, a scope becomes urgent once it is twice its interval past its last review, and urgent scopes go before every other scope that has been reviewed. When a window's slots cover the scopes that come due, every scope is reviewed within about twice its interval; when they do not, the coverage review shows it and a person widens the window.", "churn": "Changed lines under a repository path, changed page digests for a site page, or changed renders for a video segment, since the last review.", "model_calls": 0 } }, { "slug": "diff-rules", "category": "noun.reflex_procedure", "role": "diff_rules", "title": "Sorting this run's findings against the last run's", "category_reason": "A zero-call rule: the same two runs always sort the same way.", "value": { "match_by": "The findings to fixes fingerprint, so a finding keeps its identity across runs.", "seen_now": { "still_open": "Seen by the last run and again now, with no verified fix.", "reopened": "Closed as resolved before, or open with a fix a different worker verified, and seen again now: the fix did not hold and its guard missed it.", "decided": "Closed before by a person's decision, as won't fix or duplicate, and seen again: listed under decided_seen with that decision, never reopened or proposed for a fix.", "new": "Never seen before." }, "not_seen_now": { "not_checked": "Its check did not run this time, or the check's version changed since the last run; it stays open.", "fixed": "A fix is recorded and a worker other than the fixer verified it.", "gone_unverified": "A fix is recorded but not yet verified by another worker.", "gone_unexplained": "No fix is recorded; it may be hidden, moved or intermittent." }, "first_run": "The first run of a scope, and only that run, carries nothing from a previous run: every finding is new.", "follow_ups": { "new": { "$verb": "findings-to-fixes.record-finding" }, "still_open": { "$verb": "findings-to-fixes.propose-fix" }, "gone_unverified": { "$verb": "findings-to-fixes.verify-fix" }, "gone_unexplained": { "$verb": "findings-to-fixes.reproduce" }, "reopened": { "$verb": "findings-to-fixes.triage" }, "reopened_guard": { "$verb": "findings-to-fixes.add-regression-guard" } }, "model_calls": 0 } }, { "slug": "scope-kinds", "category": "noun.skill_pack", "role": "scope_kinds", "title": "Cutting a repository, site or video into scopes, and the checks each kind needs", "category_reason": "A reusable description of how to scope a review and which packs' checks to run.", "value": { "kinds": [ { "kind": "repo_path", "cut": "A folder or package that one worker can review in one sitting.", "checks": [ { "$verb": "codebase-health-review.age-todos" }, { "$verb": "codebase-health-review.find-swallowed-exceptions" }, { "$verb": "codebase-health-review.check-dependencies" }, { "$verb": "wiring-and-conformance-checks.find-unused-modules" }, { "$verb": "edge-and-corner-cases.check-api-edges" }, { "$verb": "security-review.review-code" } ] }, { "kind": "site_page", "cut": "One page and its states.", "checks": [ { "$verb": "accessibility-audit.check-contrast" }, { "$verb": "screenshot-design-review.check-design-rules" }, { "$verb": "launch-readiness.check-live-links" }, { "$verb": "edge-and-corner-cases.check-interface-edges" }, { "$verb": "annoyance-hunt.hunt-annoyances" } ] }, { "kind": "video_segment", "cut": "A stretch of a cut between two scene changes, or the whole cut when it is short.", "checks": [ { "$verb": "video-qa.compile-qa-report" }, { "$verb": "video-review.review-cut" }, { "$verb": "edge-and-corner-cases.check-video-edges" } ] }, { "kind": "docs_page", "cut": "One documentation page.", "checks": [ { "$verb": "docs-improvement.check-examples" }, { "$verb": "docs-improvement.audit-docs" } ] } ], "baseline_checks": "Any scope can also add regression-watch.compare-to-baseline when it has an accepted baseline.", "rule": "Give each scope a max interval: 14 days for code and pages people use every day, 30 for documentation and videos." } }, { "slug": "self-review-rules", "category": "noun.reflex_procedure", "role": "self_review_rules", "title": "Reviewing your own change before anyone else does", "category_reason": "A zero-call rule set for self-review.", "value": { "steps": [ "Map the change's paths to the scopes they touch in the catalog.", "Run those scopes' deterministic checks on the change, and list any that cannot run.", "Record every problem found in your own work at once as a finding, even one you will fix in the next minute.", "Hand the change and the self-review to a different worker or a person; never approve it yourself." ], "never": [ "Mark your own change verified or approved.", "Leave a problem you found off the record because you fixed it." ], "model_calls": 0 } }, { "slug": "loop-checks", "category": "noun.benchmark_suite", "role": "record_checks", "title": "Record checks for review runs, with synthetic golden runs", "category_reason": "Deterministic checks with synthetic known-good and known-bad records; the validator recomputes every verdict.", "value": { "fixture_kind": "finding_checks", "purpose": "Checks every review run must pass before it is appended to the log: the pinned scope and exact version, a person's approval for a scheduled run or a named requester for a manual one, the run it is compared with, checks that ran with their outputs and checks that did not with reasons, and every finding sorted with its follow-up: new ones recorded, fixed ones verified by someone other than the fixer, unverified and unexplained ones given a verification or a reproduction, unchecked ones kept open and reopened ones naming the closing they undid. The two synthetic good runs pass; each synthetic bad one fails exactly the one check named in fails.", "deterministic_checks": [ { "id": "scope-pinned", "rule": { "rule": "pattern_present", "path": "scope_ref.value_sha256", "pattern": "^[0-9a-f]{64}$" }, "why": "The run pins the exact scope it reviewed." }, { "id": "version-recorded", "rule": { "rule": "present", "path": "target_version" }, "why": "The run names the exact version it reviewed." }, { "id": "schedule-approved", "rule": { "rule": "when", "if": { "path": "trigger", "equals": "schedule" }, "then": { "rule": "present", "path": "approval_ref" } }, "why": "A scheduled run names the person's approval it ran under." }, { "id": "manual-requested", "rule": { "rule": "when", "if": { "path": "trigger", "equals": "manual" }, "then": { "rule": "present", "path": "requested_by" } }, "why": "A manual run names who asked for it." }, { "id": "previous-run-named", "rule": { "rule": "when", "if": { "path": "first_run", "equals": false }, "then": { "rule": "present", "path": "previous_run_ref" } }, "why": "A run that is not the first names the run it is compared with." }, { "id": "first-run-only-new", "rule": { "rule": "when", "if": { "path": "first_run", "equals": true }, "then": { "rule": "enum", "path": "carried", "values": [ 0 ] } }, "why": "A first run carries nothing from a previous run, so it cannot hide findings by calling itself first." }, { "id": "checks-listed", "rule": { "rule": "each_has", "path": "checks_run", "fields": [ "check", "result", "output_sha256" ], "min": 1 }, "why": "Every check that ran gives its result and the digest of its output, so a claimed run traces to what it produced." }, { "id": "not-run-listed-apart", "rule": { "rule": "pattern_absent", "path": "checks_run", "pattern": "^(not_run|skipped|unknown)$" }, "why": "A check marked not run, skipped or unknown is listed under checks_not_run, never among the checks that ran." }, { "id": "not-run-explained", "rule": { "rule": "each_has", "path": "checks_not_run", "fields": [ "check", "reason" ] }, "why": "Every check that did not run says why." }, { "id": "new-recorded", "rule": { "rule": "each_has", "path": "diff.new", "fields": [ "fingerprint", "finding_ref" ] }, "why": "Every new finding is recorded." }, { "id": "fixed-verified", "rule": { "rule": "each_has", "path": "diff.fixed", "fields": [ "fingerprint", "fix_ref", "verification_ref", "verified_by" ] }, "why": "A finding is fixed only with a fix, a verification and who verified it." }, { "id": "verifier-not-fixer", "rule": { "rule": "pattern_absent", "path": "diff.fixed", "pattern": "^(fixer|same_worker|self)$" }, "why": "The worker that made a fix never verifies it." }, { "id": "unverified-gets-verification", "rule": { "rule": "each_has", "path": "diff.gone_unverified", "fields": [ "fingerprint", "fix_ref", "proposed_verb" ] }, "why": "A fix nobody verified yet carries its proposed verb, verify-fix by the diff rules; diff-against-last-run's acceptance recomputes which." }, { "id": "unexplained-gets-reproduce", "rule": { "rule": "each_has", "path": "diff.gone_unexplained", "fields": [ "fingerprint", "proposed_verb" ] }, "why": "A finding gone with no fix carries its proposed verb, reproduce by the diff rules." }, { "id": "not-checked-stays-open", "rule": { "rule": "each_has", "path": "diff.not_checked", "fields": [ "fingerprint", "check" ] }, "why": "A finding whose check did not run stays open, naming the check." }, { "id": "reopened-names-closure", "rule": { "rule": "each_has", "path": "diff.reopened", "fields": [ "fingerprint", "closed_ref", "proposed_verb" ] }, "why": "A reopened finding names the closing or verification it undid and its next verb, triage by the diff rules." } ], "golden_fixtures": [ { "id": "good-scheduled", "label": "known_good", "provenance": "synthetic", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "good-first-manual", "label": "known_good", "provenance": "synthetic", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "manual", "first_run": true, "previous_run_ref": null, "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [], "fixed": [], "gone_unverified": [], "gone_unexplained": [], "not_checked": [], "reopened": [] }, "decided_seen": [], "carried": 0, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z", "requested_by": "site owner (a person)" } }, { "id": "bad-scope-unpinned", "label": "known_bad", "fails": "scope-pinned", "why": "No digest for the scope.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "current" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-no-version", "label": "known_bad", "fails": "version-recorded", "why": "No version.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-unapproved-schedule", "label": "known_bad", "fails": "schedule-approved", "why": "A scheduled run with no approval.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-manual-anonymous", "label": "known_bad", "fails": "manual-requested", "why": "A manual run nobody asked for.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "manual", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-no-previous", "label": "known_bad", "fails": "previous-run-named", "why": "Compared with nothing named.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": null, "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-first-run-carries", "label": "known_bad", "fails": "first-run-only-new", "why": "Calls itself a first run while carrying findings, so no previous run is named.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": true, "previous_run_ref": null, "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-check-no-output", "label": "known_bad", "fails": "checks-listed", "why": "A check result with no output digest.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-not-run-as-run", "label": "known_bad", "fails": "not-run-listed-apart", "why": "A check that did not run listed as run.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "not_run", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-not-run-silent", "label": "known_bad", "fails": "not-run-explained", "why": "A check not run with no reason.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps" } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-new-unrecorded", "label": "known_bad", "fails": "new-recorded", "why": "A new finding never recorded.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-gone-called-fixed", "label": "known_bad", "fails": "fixed-verified", "why": "Called fixed because it was not seen, with no verification or verifier.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-fixer-verified", "label": "known_bad", "fails": "verifier-not-fixer", "why": "The fixer verified its own fix.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "fixer" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-unverified-dropped", "label": "known_bad", "fails": "unverified-gets-verification", "why": "An unverified fix with no verification proposed.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-gone-ignored", "label": "known_bad", "fails": "unexplained-gets-reproduce", "why": "A finding gone with no fix and no reproduction proposed.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-not-checked-closed", "label": "known_bad", "fails": "not-checked-stays-open", "why": "A finding whose check did not run, with the check dropped.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps" } ], "reopened": [ { "fingerprint": "f/a/back", "closed_ref": "close.t-7", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } }, { "id": "bad-reopened-silent", "label": "known_bad", "fails": "reopened-names-closure", "why": "A reopened finding with no closing named.", "finding": { "run_id": "RR-90", "scope_id": "repo:src/api", "scope_ref": { "store": "s.o", "key": "catalog.t-1", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1", "first_run": false, "previous_run_ref": "RR-80", "checks_run": [ { "check": "x.find-swallowed", "result": "fail", "output_sha256": "2222222222222222222222222222222222222222222222222222222222222222" } ], "checks_not_run": [ { "check": "x.check-deps", "reason": "Registry unreachable." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "f/a/new", "finding_ref": "finding.t-3" } ], "still_open": [ { "fingerprint": "f/a/open", "finding_ref": "finding.t-1" } ], "fixed": [ { "fingerprint": "f/a/fixed", "fix_ref": "fix.t-4", "verification_ref": "verify.t-5", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "f/a/unv", "fix_ref": "fix.t-6", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "f/a/gone", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "f/a/deps", "check": "x.check-deps" } ], "reopened": [ { "fingerprint": "f/a/back", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [], "carried": 6, "reviewed_by": "loop worker", "reviewed_at": "2026-10-30T10:00Z" } } ], "expected": { "good-scheduled": { "verdict": "pass", "failed_checks": [] }, "good-first-manual": { "verdict": "pass", "failed_checks": [] }, "bad-scope-unpinned": { "verdict": "fail", "failed_checks": [ "scope-pinned" ] }, "bad-no-version": { "verdict": "fail", "failed_checks": [ "version-recorded" ] }, "bad-unapproved-schedule": { "verdict": "fail", "failed_checks": [ "schedule-approved" ] }, "bad-manual-anonymous": { "verdict": "fail", "failed_checks": [ "manual-requested" ] }, "bad-no-previous": { "verdict": "fail", "failed_checks": [ "previous-run-named" ] }, "bad-first-run-carries": { "verdict": "fail", "failed_checks": [ "first-run-only-new" ] }, "bad-check-no-output": { "verdict": "fail", "failed_checks": [ "checks-listed" ] }, "bad-not-run-as-run": { "verdict": "fail", "failed_checks": [ "not-run-listed-apart" ] }, "bad-not-run-silent": { "verdict": "fail", "failed_checks": [ "not-run-explained" ] }, "bad-new-unrecorded": { "verdict": "fail", "failed_checks": [ "new-recorded" ] }, "bad-gone-called-fixed": { "verdict": "fail", "failed_checks": [ "fixed-verified" ] }, "bad-fixer-verified": { "verdict": "fail", "failed_checks": [ "verifier-not-fixer" ] }, "bad-unverified-dropped": { "verdict": "fail", "failed_checks": [ "unverified-gets-verification" ] }, "bad-gone-ignored": { "verdict": "fail", "failed_checks": [ "unexplained-gets-reproduce" ] }, "bad-not-checked-closed": { "verdict": "fail", "failed_checks": [ "not-checked-stays-open" ] }, "bad-reopened-silent": { "verdict": "fail", "failed_checks": [ "reopened-names-closure" ] } } } }, { "slug": "template.scope-catalog", "category": "noun.seed_hypothesis", "role": "template", "title": "Template: a scope catalog: the target cut into scopes, each with its checks, its max interval and how its churn is measured", "category_reason": "A catalog is a falsifiable claim about what a full review covers.", "value": { "describes": "a scope catalog: the target cut into scopes, each with its checks, its max interval and how its churn is measured", "schema": { "type": "object", "additionalProperties": false, "required": [ "catalog_id", "target", "version", "scopes" ], "properties": { "catalog_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "target": { "type": "string", "minLength": 1, "maxLength": 200 }, "version": { "type": "string", "minLength": 1, "maxLength": 120 }, "scopes": { "type": "array", "minItems": 1, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "scope_id", "kind", "selector", "checks", "max_interval_days", "churn" ], "properties": { "scope_id": { "type": "string", "minLength": 1, "maxLength": 80 }, "kind": { "type": "string", "enum": [ "repo_path", "site_page", "video_segment", "docs_page" ] }, "selector": { "type": "string", "minLength": 1, "maxLength": 200 }, "checks": { "type": "array", "minItems": 1, "maxItems": 10, "items": { "type": "string", "minLength": 1, "maxLength": 120 } }, "max_interval_days": { "type": "integer", "minimum": 1, "maximum": 365 }, "churn": { "type": "object", "additionalProperties": false, "required": [ "measure", "cap" ], "properties": { "measure": { "type": "string", "minLength": 1, "maxLength": 120 }, "cap": { "type": "number", "minimum": 0.001, "maximum": 1000000 } } } } } } } }, "rules": [ "Every scope has at least one check, named by its verb template id.", "Scope ids are unique." ] } }, { "slug": "template.review-schedule", "category": "noun.seed_hypothesis", "role": "template", "title": "Template: a review schedule proposal: cadence, scopes per window and allowance, never activated by the pack and waiting for a person's approval", "category_reason": "Tessera's research seed tooling stores a schedule proposal as a seed hypothesis; this follows that precedent.", "value": { "describes": "a review schedule proposal: cadence, scopes per window and allowance, never activated by the pack and waiting for a person's approval", "schema": { "type": "object", "additionalProperties": false, "required": [ "schedule_id", "catalog_ref", "activation", "activation_path", "cadence", "scopes_per_window", "per_window", "enqueues", "start", "stop", "approval_needed", "missed_windows", "enqueue_identity" ], "properties": { "schedule_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "catalog_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "activation": { "type": "string", "enum": [ "not_activated" ] }, "activation_path": { "type": "string", "minLength": 1, "maxLength": 400 }, "cadence": { "type": "object", "additionalProperties": false, "required": [ "every", "at_utc" ], "properties": { "every": { "type": "string", "minLength": 3, "maxLength": 20, "pattern": "^P[0-9]+[DW]$" }, "at_utc": { "type": "string", "minLength": 5, "maxLength": 5, "pattern": "^[0-2][0-9]:[0-5][0-9]$" } } }, "scopes_per_window": { "type": "integer", "minimum": 1, "maximum": 10 }, "per_window": { "type": "object", "additionalProperties": false, "required": [ "max_runtime_seconds", "max_model_calls", "max_outstanding" ], "properties": { "max_runtime_seconds": { "type": "integer", "minimum": 60, "maximum": 86400 }, "max_model_calls": { "type": [ "integer", "null" ], "minimum": 0, "maximum": 100000 }, "max_outstanding": { "type": "integer", "minimum": 1, "maximum": 10 } } }, "enqueues": { "type": "array", "minItems": 1, "maxItems": 1, "items": { "type": "string", "minLength": 1, "maxLength": 120 } }, "start": { "type": [ "string", "null" ], "maxLength": 30 }, "stop": { "type": [ "string", "null" ], "maxLength": 30 }, "approval_needed": { "type": "boolean", "enum": [ true ] }, "missed_windows": { "type": "string", "minLength": 1, "maxLength": 200 }, "enqueue_identity": { "type": "string", "minLength": 1, "maxLength": 200 } } }, "rules": [ "activation stays not_activated; start and stop stay null until a person's approval sets them.", "enqueues names pick-scope only; the checks are proposed by each pick." ] } }, { "slug": "template.scope-pick", "category": "noun.eval_run", "role": "template", "title": "Template: one pick: every scope's days since review, churn, open high-severity findings, score and rank, and the scopes picked", "category_reason": "A pick is a recorded run of the rotation rule.", "value": { "describes": "one pick: every scope's days since review, churn, open high-severity findings, score and rank, and the scopes picked", "schema": { "type": "object", "additionalProperties": false, "required": [ "pick_id", "catalog_ref", "log_ref", "as_of", "scopes_per_window", "candidates", "picked" ], "properties": { "pick_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "catalog_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "log_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "as_of": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "scopes_per_window": { "type": "integer", "minimum": 1, "maximum": 10 }, "candidates": { "type": "array", "minItems": 1, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "scope_id", "last_reviewed", "days_since", "churn", "open_high", "overdue", "urgent", "score", "rank" ], "properties": { "scope_id": { "type": "string", "minLength": 1, "maxLength": 80 }, "last_reviewed": { "type": [ "string", "null" ], "maxLength": 30 }, "days_since": { "type": [ "integer", "null" ], "minimum": 0, "maximum": 100000 }, "churn": { "type": "number", "minimum": 0, "maximum": 100000000 }, "open_high": { "type": "integer", "minimum": 0, "maximum": 1000 }, "overdue": { "type": "boolean" }, "urgent": { "type": "boolean" }, "score": { "type": [ "number", "null" ], "minimum": 0, "maximum": 10000 }, "rank": { "type": "integer", "minimum": 1, "maximum": 60 } } } }, "picked": { "type": "array", "minItems": 0, "maxItems": 10, "items": { "type": "string", "minLength": 1, "maxLength": 80 } } } }, "rules": [ "Recomputing the rotation rule from the catalog and log gives the same ranks." ] } }, { "slug": "template.review-run", "category": "noun.eval_run", "role": "template", "title": "Template: one review run of one scope: the exact version, the checks that ran and did not, and every finding sorted against the last run", "category_reason": "A review run is a recorded verification run.", "value": { "describes": "one review run of one scope: the exact version, the checks that ran and did not, and every finding sorted against the last run", "schema": { "type": "object", "additionalProperties": false, "required": [ "run_id", "scope_id", "scope_ref", "target_version", "trigger", "first_run", "previous_run_ref", "checks_run", "checks_not_run", "checks_changed", "diff", "decided_seen", "carried", "reviewed_by", "reviewed_at" ], "properties": { "run_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "scope_id": { "type": "string", "minLength": 1, "maxLength": 80 }, "scope_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "target_version": { "type": "string", "minLength": 1, "maxLength": 120 }, "trigger": { "type": "string", "enum": [ "schedule", "manual" ] }, "approval_ref": { "type": "string", "minLength": 1, "maxLength": 200 }, "requested_by": { "type": "string", "minLength": 1, "maxLength": 120 }, "first_run": { "type": "boolean" }, "previous_run_ref": { "type": [ "string", "null" ], "maxLength": 200 }, "checks_run": { "type": "array", "minItems": 1, "maxItems": 20, "items": { "type": "object", "additionalProperties": false, "required": [ "check", "result", "output_sha256" ], "properties": { "check": { "type": "string", "minLength": 1, "maxLength": 120 }, "result": { "type": "string", "enum": [ "pass", "fail" ] }, "output_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } } }, "checks_not_run": { "type": "array", "minItems": 0, "maxItems": 20, "items": { "type": "object", "additionalProperties": false, "required": [ "check", "reason" ], "properties": { "check": { "type": "string", "minLength": 1, "maxLength": 120 }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "checks_changed": { "type": "array", "minItems": 0, "maxItems": 20, "items": { "type": "string", "minLength": 1, "maxLength": 120 } }, "diff": { "type": "object", "additionalProperties": false, "required": [ "new", "still_open", "fixed", "gone_unverified", "gone_unexplained", "not_checked", "reopened" ], "properties": { "new": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "finding_ref" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "finding_ref": { "type": "string", "minLength": 1, "maxLength": 200 } } } }, "still_open": { "type": "array", "minItems": 0, "maxItems": 80, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "finding_ref" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "finding_ref": { "type": "string", "minLength": 1, "maxLength": 200 } } } }, "fixed": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "fix_ref", "verification_ref", "verified_by" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "fix_ref": { "type": "string", "minLength": 1, "maxLength": 200 }, "verification_ref": { "type": "string", "minLength": 1, "maxLength": 200 }, "verified_by": { "type": "string", "enum": [ "different_worker", "person" ] } } } }, "gone_unverified": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "fix_ref", "proposed_verb" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "fix_ref": { "type": "string", "minLength": 1, "maxLength": 200 }, "proposed_verb": { "type": "string", "minLength": 1, "maxLength": 120 } } } }, "gone_unexplained": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "proposed_verb" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "proposed_verb": { "type": "string", "minLength": 1, "maxLength": 120 } } } }, "not_checked": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "check" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "check": { "type": "string", "minLength": 1, "maxLength": 120 } } } }, "reopened": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "closed_ref", "proposed_verb" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "closed_ref": { "type": "string", "minLength": 1, "maxLength": 200 }, "proposed_verb": { "type": "string", "minLength": 1, "maxLength": 120 } } } } } }, "decided_seen": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "decision_ref" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "decision_ref": { "type": "string", "minLength": 1, "maxLength": 200 } } } }, "carried": { "type": "integer", "minimum": 0, "maximum": 100000 }, "reviewed_by": { "type": "string", "minLength": 1, "maxLength": 120 }, "reviewed_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 } } }, "rules": [ "Passes loop-checks.", "carried counts every finding in the diff that is not new.", "log-run counts each diff list when it appends the run." ] } }, { "slug": "template.review-log", "category": "noun.eval_run", "role": "template", "title": "Template: the review log: every run appended in order, and coverage as of a date", "category_reason": "The log is the record of every run.", "value": { "describes": "the review log: every run appended in order, and coverage as of a date", "schema": { "type": "object", "additionalProperties": false, "required": [ "log_id", "catalog_ref", "as_of", "entries", "coverage", "open_findings" ], "properties": { "log_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "catalog_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "as_of": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "entries": { "type": "array", "minItems": 0, "maxItems": 2000, "items": { "type": "object", "additionalProperties": false, "required": [ "run_id", "scope_id", "reviewed_at", "target_version", "trigger", "counts" ], "properties": { "run_id": { "type": "string", "minLength": 1, "maxLength": 60 }, "scope_id": { "type": "string", "minLength": 1, "maxLength": 80 }, "reviewed_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "target_version": { "type": "string", "minLength": 1, "maxLength": 120 }, "trigger": { "type": "string", "enum": [ "schedule", "manual" ] }, "counts": { "type": "object", "additionalProperties": false, "required": [ "new", "still_open", "fixed", "gone_unverified", "gone_unexplained", "not_checked", "reopened" ], "properties": { "new": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "still_open": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "fixed": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "gone_unverified": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "gone_unexplained": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "not_checked": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "reopened": { "type": "integer", "minimum": 0, "maximum": 1000000 } } } } } }, "coverage": { "type": "object", "additionalProperties": false, "required": [ "scopes_total", "reviewed_within_interval", "never_reviewed", "overdue" ], "properties": { "scopes_total": { "type": "integer", "minimum": 1, "maximum": 1000 }, "reviewed_within_interval": { "type": "integer", "minimum": 0, "maximum": 1000 }, "never_reviewed": { "type": "array", "minItems": 0, "maxItems": 1000, "items": { "type": "string", "minLength": 1, "maxLength": 80 } }, "overdue": { "type": "array", "minItems": 0, "maxItems": 1000, "items": { "type": "string", "minLength": 1, "maxLength": 80 } } } }, "open_findings": { "type": "integer", "minimum": 0, "maximum": 100000 } } }, "rules": [ "Entries are only appended, by compare-and-set; none is edited or removed.", "coverage recomputes from the entries and the catalog." ] } }, { "slug": "template.self-review", "category": "noun.eval_run", "role": "template", "title": "Template: a self-review of one's own change before handing it on: the scopes it touched, the checks run, the findings recorded and who it went to", "category_reason": "A self-review is a recorded check run.", "value": { "describes": "a self-review of one's own change before handing it on: the scopes it touched, the checks run, the findings recorded and who it went to", "schema": { "type": "object", "additionalProperties": false, "required": [ "self_review_id", "change", "changed_paths", "scopes_touched", "checks_run", "checks_not_run", "own_findings", "handed_to", "self_approval" ], "properties": { "self_review_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "change": { "type": "string", "minLength": 1, "maxLength": 200 }, "changed_paths": { "type": "array", "minItems": 1, "maxItems": 200, "items": { "type": "string", "minLength": 1, "maxLength": 200 } }, "scopes_touched": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "string", "minLength": 1, "maxLength": 80 } }, "checks_run": { "type": "array", "minItems": 0, "maxItems": 20, "items": { "type": "object", "additionalProperties": false, "required": [ "check", "result", "output_ref" ], "properties": { "check": { "type": "string", "minLength": 1, "maxLength": 120 }, "result": { "type": "string", "enum": [ "pass", "fail" ] }, "output_ref": { "type": "string", "minLength": 1, "maxLength": 200 } } } }, "checks_not_run": { "type": "array", "minItems": 0, "maxItems": 20, "items": { "type": "object", "additionalProperties": false, "required": [ "check", "reason" ], "properties": { "check": { "type": "string", "minLength": 1, "maxLength": 120 }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "own_findings": { "type": "array", "minItems": 0, "maxItems": 40, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "finding_ref", "fixed_in_change" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "finding_ref": { "type": "string", "minLength": 1, "maxLength": 200 }, "fixed_in_change": { "type": "boolean" } } } }, "handed_to": { "type": "string", "minLength": 1, "maxLength": 120 }, "self_approval": { "type": "boolean", "enum": [ false ] } } }, "rules": [ "self_approval is always false.", "Every problem found is in own_findings, fixed or not." ] } }, { "slug": "template.coverage-review", "category": "noun.eval_run", "role": "template", "title": "Template: a coverage review across a window: scopes left out, checks that never fail or rarely run, causes that keep coming back, and proposals", "category_reason": "A coverage review is a recorded analysis run.", "value": { "describes": "a coverage review across a window: scopes left out, checks that never fail or rarely run, causes that keep coming back, and proposals", "schema": { "type": "object", "additionalProperties": false, "required": [ "review_id", "log_ref", "window", "never_reviewed", "overdue_twice", "checks_never_failing", "checks_often_not_run", "recurring", "proposals" ], "properties": { "review_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "log_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "window": { "type": "object", "additionalProperties": false, "required": [ "from", "to" ], "properties": { "from": { "type": "string", "minLength": 10, "maxLength": 10 }, "to": { "type": "string", "minLength": 10, "maxLength": 10 } } }, "never_reviewed": { "type": "array", "minItems": 0, "maxItems": 1000, "items": { "type": "string", "minLength": 1, "maxLength": 80 } }, "overdue_twice": { "type": "array", "minItems": 0, "maxItems": 1000, "items": { "type": "string", "minLength": 1, "maxLength": 80 } }, "checks_never_failing": { "type": "array", "minItems": 0, "maxItems": 200, "items": { "type": "string", "minLength": 1, "maxLength": 120 } }, "checks_often_not_run": { "type": "array", "minItems": 0, "maxItems": 200, "items": { "type": "object", "additionalProperties": false, "required": [ "check", "not_run", "runs" ], "properties": { "check": { "type": "string", "minLength": 1, "maxLength": 120 }, "not_run": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "runs": { "type": "integer", "minimum": 0, "maximum": 1000000 } } } }, "recurring": { "type": "array", "minItems": 0, "maxItems": 200, "items": { "type": "object", "additionalProperties": false, "required": [ "fingerprint", "reopened" ], "properties": { "fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 }, "reopened": { "type": "integer", "minimum": 1, "maximum": 1000 } } } }, "proposals": { "type": "array", "minItems": 0, "maxItems": 10, "items": { "type": "object", "additionalProperties": false, "required": [ "what", "verb" ], "properties": { "what": { "type": "string", "minLength": 1, "maxLength": 300 }, "verb": { "type": "string", "minLength": 1, "maxLength": 120 } } } } } }, "rules": [ "A proposal that changes the catalog or schedule goes back to a person for a new approval." ] } }, { "slug": "test-cases", "category": "noun.benchmark_suite", "role": "test_cases", "title": "Synthetic test cases for continuous self-review", "synthetic": true, "value": { "cases": [ { "id": "CSR-NO-APPROVAL", "scenario": "A window opens before any person approved the schedule", "sample_input": "SYNTHETIC: schedule RS-1 with no approval.", "expected_outcome": "blocked", "expected_indicators": [ "approval_needed" ], "expected_human_escalation": true }, { "id": "CSR-EXPIRED", "scenario": "The approval expired yesterday", "sample_input": "SYNTHETIC: AP-1 expired 2026-12-31; today is 2027-01-01.", "expected_outcome": "blocked", "expected_indicators": [ "approval_expired", "new_approval_needed" ], "expected_human_escalation": true }, { "id": "CSR-GONE", "scenario": "A finding disappears with no fix on record", "sample_input": "SYNTHETIC: the TODO at line 88 is no longer reported.", "expected_outcome": "run_diffed", "expected_indicators": [ "gone_unexplained", "reproduce_proposed", "not_fixed" ], "expected_human_escalation": false }, { "id": "CSR-CHECK-DOWN", "scenario": "A check cannot run this window", "sample_input": "SYNTHETIC: registry metadata unreachable.", "expected_outcome": "run_diffed", "expected_indicators": [ "not_checked", "stays_open" ], "expected_human_escalation": false }, { "id": "CSR-SELF-VERIFY", "scenario": "The worker that fixed a finding is asked to verify it", "sample_input": "SYNTHETIC: the fixer's key claims verify-fix.", "expected_outcome": "blocked", "expected_indicators": [ "distinct_reviewer_required" ], "expected_human_escalation": false }, { "id": "CSR-STARVED", "scenario": "Busy scopes keep outranking a quiet one", "sample_input": "SYNTHETIC: three busy scopes with open P1 findings and a quiet docs scope 61 days since review on a 30-day interval.", "expected_outcome": "scopes_picked", "expected_indicators": [ "urgent_tier", "quiet_scope_picked_first" ], "expected_human_escalation": false }, { "id": "CSR-WONT-FIX-SEEN", "scenario": "A finding a person closed as won't fix is seen again", "sample_input": "SYNTHETIC: the cache count's broad suppress.", "expected_outcome": "run_diffed", "expected_indicators": [ "decided_seen", "no_fix_proposed" ], "expected_human_escalation": false } ] } }, { "slug": "example.scope-catalog", "category": "noun.seed_hypothesis", "role": "example", "title": "Worked example: eight scopes across a repository, a site and a video", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.scope-catalog" }, "instance": { "catalog_id": "SC-1", "target": "SYNTHETIC pantry: its repository, its site and its 45-second explainer", "version": "1", "scopes": [ { "scope_id": "repo:src/api", "kind": "repo_path", "selector": "src/api/**", "checks": [ "codebase-health-review.find-swallowed-exceptions", "codebase-health-review.age-todos", "edge-and-corner-cases.check-api-edges", "codebase-health-review.check-dependencies", "wiring-and-conformance-checks.find-unused-modules" ], "max_interval_days": 14, "churn": { "measure": "changed lines", "cap": 200 } }, { "scope_id": "repo:src/forms", "kind": "repo_path", "selector": "src/forms/**", "checks": [ "codebase-health-review.find-swallowed-exceptions", "codebase-health-review.age-todos", "edge-and-corner-cases.run-input-cases" ], "max_interval_days": 14, "churn": { "measure": "changed lines", "cap": 200 } }, { "scope_id": "repo:docs", "kind": "docs_page", "selector": "docs/**", "checks": [ "docs-improvement.check-examples" ], "max_interval_days": 30, "churn": { "measure": "changed lines", "cap": 200 } }, { "scope_id": "page:/", "kind": "site_page", "selector": "/", "checks": [ "accessibility-audit.check-contrast", "launch-readiness.check-live-links" ], "max_interval_days": 30, "churn": { "measure": "changed page digests", "cap": 1 } }, { "scope_id": "page:/shifts", "kind": "site_page", "selector": "/shifts", "checks": [ "accessibility-audit.check-contrast", "screenshot-design-review.check-design-rules", "annoyance-hunt.hunt-annoyances" ], "max_interval_days": 14, "churn": { "measure": "changed page digests", "cap": 5 } }, { "scope_id": "page:/signup", "kind": "site_page", "selector": "/signup", "checks": [ "accessibility-audit.check-contrast", "edge-and-corner-cases.check-interface-edges", "annoyance-hunt.hunt-annoyances" ], "max_interval_days": 14, "churn": { "measure": "changed page digests", "cap": 5 } }, { "scope_id": "video:explainer-0-15", "kind": "video_segment", "selector": "explainer 0.0 to 15.0 s", "checks": [ "video-qa.compile-qa-report", "edge-and-corner-cases.check-video-edges" ], "max_interval_days": 30, "churn": { "measure": "changed renders", "cap": 1 } }, { "scope_id": "video:explainer-15-45", "kind": "video_segment", "selector": "explainer 15.0 to 45.0 s", "checks": [ "video-qa.compile-qa-report", "video-review.review-cut" ], "max_interval_days": 30, "churn": { "measure": "changed renders", "cap": 1 } } ] }, "shows": "Each scope with checks from other packs, a max interval and how its churn is measured.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.review-schedule", "category": "noun.seed_hypothesis", "role": "example", "title": "Worked example: a weekly rotation, proposed and not activated", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.review-schedule" }, "instance": { "schedule_id": "RS-1", "catalog_ref": { "$pin": "example.scope-catalog" }, "activation": "not_activated", "activation_path": "Recurring execution needs Tessera's flag-gated schedule support and a person's approval. No scheduler is activated by this pack, and a schedule noun cannot authorize work by itself.", "cadence": { "every": "P1W", "at_utc": "06:00" }, "scopes_per_window": 3, "per_window": { "max_runtime_seconds": 7200, "max_model_calls": 60, "max_outstanding": 1 }, "enqueues": [ "continuous-self-review.pick-scope" ], "start": null, "stop": null, "approval_needed": true, "missed_windows": "Skip missed windows; never catch up more than one.", "enqueue_identity": "SHA-256 of the schedule key, its version and the window start." }, "shows": "Three scopes a week, an allowance per window, pick-scope as the only verb it would enqueue, and no start until a person approves.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.schedule-approval", "category": "noun.eval_run", "role": "example", "title": "Worked example: a person approves the rotation with changes and an expiry", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "unattended-run-record:template.run-approval" }, "instance": { "approval_id": "AP-1", "plan_ref": "review schedule RS-1, version 1", "decision": "approved_with_changes", "scope": [ "repo:src/api", "repo:src/forms", "repo:docs", "page:/", "page:/shifts", "page:/signup", "video:explainer-0-15", "video:explainer-15-45" ], "allowance": "Up to three scopes a week, two hours and sixty model calls per window, from the requester's own allowance.", "changes": [ "Start on Friday 2026-10-23 at 06:00 UTC and repeat weekly.", "Stop on 2026-12-31 unless renewed." ], "decided_by_role": "site owner", "decided_at": "2026-10-19T09:00Z", "expires": "2026-12-31T23:59Z" }, "shows": "The unattended run record pack's approval, reused: scopes, allowance, changes and an expiry the scheduled runs name.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.scope-pick", "category": "noun.eval_run", "role": "example", "title": "Worked example: the rotation rule picks three scopes", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.scope-pick" }, "instance": { "pick_id": "SP-6", "catalog_ref": { "$pin": "example.scope-catalog" }, "log_ref": { "$pin": "example.review-log" }, "as_of": "2026-10-30T06:00Z", "scopes_per_window": 3, "candidates": [ { "scope_id": "page:/signup", "last_reviewed": null, "days_since": null, "churn": 0, "open_high": 0, "overdue": false, "urgent": false, "score": null, "rank": 1 }, { "scope_id": "repo:src/api", "last_reviewed": "2026-10-23T06:30Z", "days_since": 7, "churn": 420, "open_high": 1, "overdue": false, "urgent": false, "score": 2.0, "rank": 2 }, { "scope_id": "repo:docs", "last_reviewed": "2026-09-02T06:30Z", "days_since": 58, "churn": 12, "open_high": 0, "overdue": true, "urgent": false, "score": 1.993, "rank": 3 }, { "scope_id": "video:explainer-15-45", "last_reviewed": "2026-10-20T07:30Z", "days_since": 10, "churn": 1, "open_high": 1, "overdue": false, "urgent": false, "score": 1.833, "rank": 4 }, { "scope_id": "repo:src/forms", "last_reviewed": "2026-10-09T06:40Z", "days_since": 21, "churn": 60, "open_high": 0, "overdue": true, "urgent": false, "score": 1.8, "rank": 5 }, { "scope_id": "page:/shifts", "last_reviewed": "2026-10-16T07:00Z", "days_since": 14, "churn": 3, "open_high": 0, "overdue": false, "urgent": false, "score": 1.6, "rank": 6 }, { "scope_id": "page:/", "last_reviewed": "2026-10-16T06:20Z", "days_since": 14, "churn": 0, "open_high": 0, "overdue": false, "urgent": false, "score": 0.467, "rank": 7 }, { "scope_id": "video:explainer-0-15", "last_reviewed": "2026-10-20T06:30Z", "days_since": 10, "churn": 0, "open_high": 0, "overdue": false, "urgent": false, "score": 0.333, "rank": 8 } ], "picked": [ "page:/signup", "repo:src/api", "repo:docs" ] }, "shows": "A scope never reviewed first, then the highest scores: an API folder that changed most and documentation overdue by four weeks.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.diff-inputs", "category": "noun.benchmark_suite", "role": "fixture", "title": "Worked example inputs: what the last run saw, what this run saw, and the fixes and verifications on record", "synthetic": true, "category_reason": "A golden fixture: inputs a test sorts again.", "value": { "purpose": "The library's tests sort these by the diff rules and compare with the review run's diff.", "scope_id": "repo:src/api", "previous_run": "RR-9", "checks_run": [ "codebase-health-review.find-swallowed-exceptions", "codebase-health-review.age-todos", "edge-and-corner-cases.check-api-edges", "wiring-and-conformance-checks.find-unused-modules" ], "checks_not_run": [ { "check": "codebase-health-review.check-dependencies", "reason": "Public registry metadata was unreachable from the sandbox this window; it runs next window." } ], "previous_open": [ { "fingerprint": "design_flaw/roster-sync/swallowed-exceptions-save-errors-hidden", "check": "codebase-health-review.find-swallowed-exceptions" }, { "fingerprint": "error/signup-api/duplicate-json-keys-accepted", "check": "edge-and-corner-cases.check-api-edges" }, { "fingerprint": "error/signup-api/expired-lease-submit-accepted", "check": "edge-and-corner-cases.check-api-edges" }, { "fingerprint": "issue/signup-api/todo-age-validate-payload", "check": "codebase-health-review.age-todos" }, { "fingerprint": "issue/ci-workflow/unpinned-versions-setup-action", "check": "codebase-health-review.check-dependencies" } ], "checks_changed": [], "closed_before": [ { "fingerprint": "error/signup-api/name-stored-unnormalized", "closed_ref": "close of EC-7 on 2026-10-02", "reason": "resolved" }, { "fingerprint": "design_flaw/roster-sync/swallowed-exceptions-cached-count", "closed_ref": "the site owner's decision on 2026-10-10: a missing cache may count as zero", "reason": "wont_fix" } ], "seen_now": { "codebase-health-review.find-swallowed-exceptions": [ "design_flaw/roster-sync/swallowed-exceptions-save-errors-hidden", "design_flaw/roster-sync/swallowed-exceptions-remote-rows", "design_flaw/roster-sync/swallowed-exceptions-cached-count" ], "edge-and-corner-cases.check-api-edges": [ "error/signup-api/name-stored-unnormalized" ], "codebase-health-review.age-todos": [], "wiring-and-conformance-checks.find-unused-modules": [] }, "fixes": [ { "fingerprint": "error/signup-api/duplicate-json-keys-accepted", "fix_ref": "fix FX-12, a strict JSON parser" }, { "fingerprint": "error/signup-api/expired-lease-submit-accepted", "fix_ref": "fix FX-14, fencing token checked" } ], "verifications": [ { "fingerprint": "error/signup-api/duplicate-json-keys-accepted", "verification_ref": "verification VF-12", "verified_by": "different_worker" } ], "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.review-run", "category": "noun.eval_run", "role": "example", "title": "Worked example: one run of an API folder, sorted against the last run", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.review-run" }, "instance": { "run_id": "RR-12", "scope_id": "repo:src/api", "scope_ref": { "$pin": "example.scope-catalog" }, "target_version": "commit 4f2a9c1", "trigger": "schedule", "approval_ref": "approval AP-1, expires 2026-12-31", "first_run": false, "previous_run_ref": "RR-9", "checks_run": [ { "check": "codebase-health-review.find-swallowed-exceptions", "result": "fail", "output_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, { "check": "codebase-health-review.age-todos", "result": "pass", "output_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, { "check": "edge-and-corner-cases.check-api-edges", "result": "fail", "output_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" }, { "check": "wiring-and-conformance-checks.find-unused-modules", "result": "pass", "output_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" } ], "checks_not_run": [ { "check": "codebase-health-review.check-dependencies", "reason": "Public registry metadata was unreachable from the sandbox this window; it runs next window." } ], "checks_changed": [], "diff": { "new": [ { "fingerprint": "design_flaw/roster-sync/swallowed-exceptions-remote-rows", "finding_ref": "finding CH-9, recorded by this run" } ], "still_open": [ { "fingerprint": "design_flaw/roster-sync/swallowed-exceptions-save-errors-hidden", "finding_ref": "finding CH-1" } ], "fixed": [ { "fingerprint": "error/signup-api/duplicate-json-keys-accepted", "fix_ref": "fix FX-12, a strict JSON parser", "verification_ref": "verification VF-12", "verified_by": "different_worker" } ], "gone_unverified": [ { "fingerprint": "error/signup-api/expired-lease-submit-accepted", "fix_ref": "fix FX-14, fencing token checked", "proposed_verb": "findings-to-fixes.verify-fix" } ], "gone_unexplained": [ { "fingerprint": "issue/signup-api/todo-age-validate-payload", "proposed_verb": "findings-to-fixes.reproduce" } ], "not_checked": [ { "fingerprint": "issue/ci-workflow/unpinned-versions-setup-action", "check": "codebase-health-review.check-dependencies" } ], "reopened": [ { "fingerprint": "error/signup-api/name-stored-unnormalized", "closed_ref": "close of EC-7 on 2026-10-02", "proposed_verb": "findings-to-fixes.triage" } ] }, "decided_seen": [ { "fingerprint": "design_flaw/roster-sync/swallowed-exceptions-cached-count", "decision_ref": "the site owner's decision on 2026-10-10: a missing cache may count as zero" } ], "carried": 6, "reviewed_by": "diff-against-last-run worker", "reviewed_at": "2026-10-30T10:00Z" }, "shows": "One finding in each class: new, still open, fixed and verified by another worker, fixed but unverified, gone with no fix, not checked because its check did not run, and reopened after it was closed; and one a person closed as won't fix, seen again and left alone.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.review-log", "category": "noun.eval_run", "role": "example", "title": "Worked example: twelve runs over two months, and coverage as of the end of the window", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.review-log" }, "instance": { "log_id": "RL-1", "catalog_ref": { "$pin": "example.scope-catalog" }, "as_of": "2026-10-30T12:00Z", "entries": [ { "run_id": "RR-3", "scope_id": "repo:docs", "reviewed_at": "2026-09-02T06:30Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 2, "still_open": 0, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-1", "scope_id": "repo:src/api", "reviewed_at": "2026-09-28T06:30Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 4, "still_open": 0, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-2", "scope_id": "page:/shifts", "reviewed_at": "2026-09-28T07:10Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 2, "still_open": 0, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-5", "scope_id": "repo:src/forms", "reviewed_at": "2026-10-09T06:40Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 2, "still_open": 1, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-4", "scope_id": "page:/", "reviewed_at": "2026-10-16T06:20Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 1, "still_open": 0, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-8", "scope_id": "page:/shifts", "reviewed_at": "2026-10-16T07:00Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 0, "still_open": 1, "fixed": 1, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-6", "scope_id": "video:explainer-0-15", "reviewed_at": "2026-10-20T06:30Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 1, "still_open": 0, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-7", "scope_id": "video:explainer-15-45", "reviewed_at": "2026-10-20T07:30Z", "target_version": "earlier build", "trigger": "manual", "counts": { "new": 2, "still_open": 0, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-9", "scope_id": "repo:src/api", "reviewed_at": "2026-10-23T06:30Z", "target_version": "commit 1c9e4d2", "trigger": "schedule", "counts": { "new": 3, "still_open": 2, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-10", "scope_id": "page:/signup", "reviewed_at": "2026-10-30T08:10Z", "target_version": "site build 2026-10-30.4", "trigger": "schedule", "counts": { "new": 3, "still_open": 0, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-11", "scope_id": "repo:docs", "reviewed_at": "2026-10-30T09:00Z", "target_version": "commit 4f2a9c1", "trigger": "schedule", "counts": { "new": 1, "still_open": 2, "fixed": 0, "gone_unverified": 0, "gone_unexplained": 0, "not_checked": 0, "reopened": 0 } }, { "run_id": "RR-12", "scope_id": "repo:src/api", "reviewed_at": "2026-10-30T10:00Z", "target_version": "commit 4f2a9c1", "trigger": "schedule", "counts": { "new": 1, "still_open": 1, "fixed": 1, "gone_unverified": 1, "gone_unexplained": 1, "not_checked": 1, "reopened": 1 } } ], "coverage": { "scopes_total": 8, "reviewed_within_interval": 7, "never_reviewed": [], "overdue": [ "repo:src/forms" ] }, "open_findings": 20 }, "shows": "Every run appended in order, with coverage recomputed from the entries: seven of eight scopes within their intervals, and the forms folder overdue, so it leads the next pick.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.self-review", "category": "noun.eval_run", "role": "example", "title": "Worked example: a worker reviews its own change before handing it on", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.self-review" }, "instance": { "self_review_id": "SR-3", "change": "branch fix-name-normalization at commit 9d0e1aa", "changed_paths": [ "src/api/names.py", "src/forms/signup.js", "tests/test_names.py" ], "scopes_touched": [ "repo:src/api", "repo:src/forms" ], "checks_run": [ { "check": "codebase-health-review.find-swallowed-exceptions", "result": "pass", "output_ref": "area result AR-40" }, { "check": "edge-and-corner-cases.run-input-cases", "result": "fail", "output_ref": "case results ER-11" } ], "checks_not_run": [ { "check": "edge-and-corner-cases.check-api-edges", "reason": "The local build needs the queue service, which the sandbox lacks." } ], "own_findings": [ { "fingerprint": "error/signup-form/name-trim-drops-no-break-space", "finding_ref": "finding EC-15, recorded at once", "fixed_in_change": true } ], "handed_to": "a different review worker", "self_approval": false }, "shows": "Two scopes touched, one check failing on the worker's own change and recorded at once, one that could not run, and the change handed to a different worker.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.coverage-review", "category": "noun.eval_run", "role": "example", "title": "Worked example: two months of the log, read for what the loop misses", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.coverage-review" }, "instance": { "review_id": "CR-1", "log_ref": { "$pin": "example.review-log" }, "window": { "from": "2026-09-01", "to": "2026-10-30" }, "never_reviewed": [], "overdue_twice": [], "checks_never_failing": [ "launch-readiness.check-live-links" ], "checks_often_not_run": [ { "check": "codebase-health-review.check-dependencies", "not_run": 2, "runs": 3 } ], "recurring": [ { "fingerprint": "error/signup-api/name-stored-unnormalized", "reopened": 1 } ], "proposals": [ { "what": "Prove the live link check can fail at all with a planted broken link.", "verb": "checks-of-checks.validate-checks" }, { "what": "Give check-dependencies a cached registry snapshot so it can run in the sandbox.", "verb": "findings-to-fixes.record-finding" }, { "what": "Find why the name normalization fix did not hold and add a guard that fails without it.", "verb": "findings-to-fixes.review-patterns" } ] }, "shows": "A check that never fails, a check that often cannot run and a fix that did not hold, each with a proposal.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } } ], "verbs": [ { "slug": "map-scopes", "kind": "verb.problemize", "title": "Cut the target into scopes and propose a schedule", "claimant": "agent", "summary": "Cut a repository, site or video into scopes one worker can review in one sitting, give each its checks from other packs, a max interval and a churn measure, and write a review schedule proposal that waits for a person.", "outcome": "A scope catalog and a schedule proposal.", "ready_outcome": "scopes_mapped", "pinned": [ "method", "scope-kinds", "guardrail.schedule-is-a-proposal" ], "slots": [ { "name": "target_brief", "category": "noun.seed_hypothesis", "description": "Pin of a brief naming the target, its owner and the budget a review may use." } ], "produces": [ { "name": "scope_catalog", "template": "template.scope-catalog" }, { "name": "review_schedule", "template": "template.review-schedule" } ], "steps": [ "Read the target: the repository tree at a commit, the site's page list or the video's scene changes.", "Cut it into scopes by the scope kinds, give each scope the checks its kind needs, and a max interval.", "Propose a cadence, scopes per window and an allowance per window that fit the requester's stated budget.", "Leave activation not_activated and start and stop empty; only a person's approval sets them." ], "acceptance": [ { "id": "catalog-matches-template", "check": "The catalog validates against template.scope-catalog's schema.", "method": "deterministic" }, { "id": "schedule-matches-template", "check": "The schedule validates against template.review-schedule's schema.", "method": "deterministic" }, { "id": "checks-exist", "check": "Every check names a verb template in this library.", "method": "deterministic" }, { "id": "scopes-fit-one-sitting", "check": "Each scope is small enough for one review run.", "method": "judgment" } ], "network": [ "read_only", "Read-only page loads of the target site's own pages to list them; repositories and videos are read from a local copy." ], "bounds": { "max_elapsed_seconds": 1800 } }, { "slug": "approve-schedule", "kind": "verb.verify_gate", "title": "Approve the review schedule (a person)", "claimant": "human", "summary": "The person who owns the target reads the schedule proposal and approves it, approves it with changes or declines it, with the scopes, the allowance and an expiry. This template performs nothing itself.", "outcome": "A schedule approval.", "ready_outcome": "schedule_decided", "model_use": "none", "pinned": [ "guardrail.schedule-is-a-proposal", "unattended-run-record:run-guards" ], "slots": [ { "name": "review_schedule", "category": "noun.seed_hypothesis", "description": "Pin of the review schedule proposal." } ], "produces": [ { "name": "approval", "template": "unattended-run-record:template.run-approval" } ], "steps": [ "Read the scopes, cadence, allowance per window and the checks each scope would run.", "Approve only what you would accept running every window until the expiry; set the expiry." ], "acceptance": [ { "id": "approval-matches-template", "check": "The approval validates against template.run-approval's schema.", "method": "deterministic" }, { "id": "expiry-set", "check": "expires is filled.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 86400 }, "follows": [ "continuous-self-review.map-scopes" ] }, { "slug": "pick-scope", "kind": "verb.recheck_fragility", "title": "Pick the next scopes by the rotation rule", "claimant": "agent", "summary": "Apply the rotation rule to the scope catalog and the review log as of now, rank every scope, pick the window's scopes, and propose their checks.", "outcome": "A scope pick, with the picked scopes' checks proposed.", "ready_outcome": "scopes_picked", "model_use": "none", "pinned": [ "rotation-rule", "guardrail.schedule-is-a-proposal" ], "slots": [ { "name": "scope_catalog", "category": "noun.seed_hypothesis", "description": "Pin of the scope catalog." }, { "name": "review_log", "category": "noun.eval_run", "description": "Pin of the review log." }, { "name": "approval", "category": "noun.eval_run", "description": "Pin of the person's schedule approval; leave empty only for a manual pick.", "required": false } ], "produces": [ { "name": "scope_pick", "template": "template.scope-pick" } ], "steps": [ "Confirm an approval covers this window and has not expired, unless the requester asked for a manual pick; otherwise end blocked.", "Measure each scope's churn since its last review and count its open P0 and P1 findings.", "Rank by the rotation rule and pick the first scopes the approval allows.", "Propose each picked scope's checks with the target version pinned, at most ten verbs; never enqueue them." ], "acceptance": [ { "id": "pick-matches-template", "check": "The pick validates against template.scope-pick's schema.", "method": "deterministic" }, { "id": "ranks-recompute", "check": "Recomputing the rotation rule from the catalog, log and churn gives the same ranks.", "method": "deterministic" }, { "id": "within-approval", "check": "The picked scopes are within the approval's scope and number.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 600 }, "ready_requires": [ "proposed_verbs" ], "follows": [ "continuous-self-review.approve-schedule" ] }, { "slug": "diff-against-last-run", "kind": "verb.verify_gate", "title": "Diff this run's findings against the last run", "claimant": "agent", "summary": "Collect the picked scope's check outputs, list the checks that did not report, and sort every finding by fingerprint against the last run of the same scope: new, still open, fixed, gone unverified, gone unexplained, not checked or reopened.", "outcome": "A review run with its diff.", "ready_outcome": "run_diffed", "model_use": "none", "pinned": [ "diff-rules", "loop-checks", "guardrail.gone-is-not-fixed", "regression-watch:guardrail.compare-like-with-like", "findings-to-fixes:kind-definitions" ], "slots": [ { "name": "scope_pick", "category": "noun.eval_run", "description": "Pin of the scope pick." }, { "name": "check_outputs", "category": "noun.eval_run", "description": "Pins of the outputs the scope's checks submitted.", "many": true, "max": 16 }, { "name": "previous_run", "category": "noun.eval_run", "description": "Pin of the last review run of the same scope, if any.", "required": false }, { "name": "findings", "category": "noun.seed_hypothesis", "description": "Pins of the scope's findings, fixes and verifications on record.", "required": false, "many": true, "max": 64 } ], "produces": [ { "name": "review_run", "template": "template.review-run" } ], "steps": [ "Use only the outputs of checks that ran on the scope's pinned version; list every other check under checks_not_run with its reason.", "Sort every finding by the diff rules; a finding is fixed only with a recorded fix and a verification by a worker other than the fixer, and one a person closed as won't fix or duplicate goes under decided_seen with that decision.", "Propose record-finding for new ones, propose-fix for still open ones without a proposal, verify-fix for unverified fixes, reproduce for unexplained ones, and triage and a stronger guard for reopened ones.", "List every check whose version changed since the last run under checks_changed; its findings that are not seen again are not checked, never fixed or gone.", "Set carried to the number of findings that are not new; a first run carries none." ], "acceptance": [ { "id": "run-matches-template", "check": "The run validates against template.review-run's schema.", "method": "deterministic" }, { "id": "loop-checks-pass", "check": "The run passes every pinned loop check.", "method": "deterministic" }, { "id": "diff-recomputes", "check": "Sorting the pinned outputs and records by the diff rules gives the same lists.", "method": "deterministic" }, { "id": "counts-match", "check": "counts equal the length of each diff list.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 900 }, "ready_requires": [ "proposed_verbs" ], "follows": [ "continuous-self-review.pick-scope" ] }, { "slug": "log-run", "kind": "verb.compile_dag", "title": "Append the run to the review log", "claimant": "agent", "summary": "Append the review run to the review log by compare-and-set and recompute coverage: scopes reviewed within their intervals, never reviewed and overdue.", "outcome": "The next version of the review log.", "ready_outcome": "run_logged", "model_use": "none", "pinned": [ "rotation-rule" ], "slots": [ { "name": "review_log", "category": "noun.eval_run", "description": "Pin of the review log to update." }, { "name": "review_run", "category": "noun.eval_run", "description": "Pin of the review run." }, { "name": "scope_catalog", "category": "noun.seed_hypothesis", "description": "Pin of the scope catalog." } ], "produces": [ { "name": "review_log", "template": "template.review-log" } ], "steps": [ "Read the log's pinned version, append the run's entry and recompute coverage from the entries and the catalog.", "Write the new version with the pinned version as the expected version; on a conflict, re-read and stop.", "Never edit or remove an earlier entry." ], "acceptance": [ { "id": "log-matches-template", "check": "The log validates against template.review-log's schema.", "method": "deterministic" }, { "id": "append-only", "check": "Every entry of the pinned version is unchanged in the new one, and exactly one entry was added.", "method": "deterministic" }, { "id": "coverage-recomputes", "check": "Coverage recomputes from the entries and the catalog.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 300 }, "update_slot": "review_log", "follows": [ "continuous-self-review.diff-against-last-run" ] }, { "slug": "review-own-change", "kind": "verb.verify_gate", "title": "Review your own change before handing it on", "claimant": "agent_or_human", "summary": "Before handing a change on, map its paths to the scopes it touches, run those scopes' deterministic checks on it, record every problem found in your own work at once, and hand it to a different worker or a person.", "outcome": "A self-review.", "ready_outcome": "self_reviewed", "pinned": [ "self-review-rules", "scope-kinds" ], "slots": [ { "name": "change", "category": "noun.code_node", "description": "Pin of the change: its commit or patch by digest." }, { "name": "scope_catalog", "category": "noun.seed_hypothesis", "description": "Pin of the scope catalog." } ], "produces": [ { "name": "self_review", "template": "template.self-review" } ], "steps": [ "Map the change's paths to the catalog's scopes.", "Run those scopes' deterministic checks on the change and list any that cannot run, with the reason.", "Propose findings-to-fixes.record-finding for every problem found, including any fixed in the same change.", "Name the different worker or person it goes to; never approve or verify your own change." ], "acceptance": [ { "id": "self-review-matches-template", "check": "The self-review validates against template.self-review's schema.", "method": "deterministic" }, { "id": "never-self-approved", "check": "self_approval is false and handed_to names someone else.", "method": "deterministic" }, { "id": "own-findings-recorded", "check": "Every failed check has a finding in own_findings.", "method": "deterministic" } ], "network": [ "sandbox_packages", "Package installs inside the sandbox to run the scopes' checks on the change; no requests to any deployed system." ], "bounds": { "max_elapsed_seconds": 1800 }, "ready_requires": [ "proposed_verbs" ] }, { "slug": "review-coverage", "kind": "verb.distill_reflex", "title": "Read the log for what the loop misses", "claimant": "agent", "summary": "Across a window of the review log, list scopes never reviewed or overdue twice, checks that never fail or often do not run, and findings that keep coming back, and propose a fix for each.", "outcome": "A coverage review with proposals.", "ready_outcome": "coverage_reviewed", "pinned": [ "rotation-rule", "diff-rules" ], "slots": [ { "name": "review_log", "category": "noun.eval_run", "description": "Pin of the review log." }, { "name": "review_runs", "category": "noun.eval_run", "description": "Pins of the runs in the window.", "required": false, "many": true, "max": 64 } ], "produces": [ { "name": "coverage_review", "template": "template.coverage-review" } ], "steps": [ "Count from the log's entries and the runs they pin only.", "Propose checks-of-checks.validate-checks for a check that never fails, a finding for a check that often cannot run, and findings-to-fixes.review-patterns for a finding reopened again.", "A change to the catalog or schedule goes back to a person through approve-schedule." ], "acceptance": [ { "id": "review-matches-template", "check": "The review validates against template.coverage-review's schema.", "method": "deterministic" }, { "id": "counts-recompute", "check": "Every count recomputes from the pinned log and runs.", "method": "deterministic" }, { "id": "proposals-fit", "check": "Each proposal addresses the gap it names.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 1200 }, "ready_requires": [ "proposed_verbs" ], "follows": [ "continuous-self-review.log-run" ] } ] }