{ "pack_id": "regression-watch", "title": "Regression watch", "version": "0.1.0", "summary": "A collection of nouns and verbs for watching for regressions: baselines kept as nouns, from page screenshots by hash to metrics with tolerances, API response shapes and video QA numbers; every change measured again under the same settings and compared by fixed rules; every mismatch measured twice more before anything is reported, so a flaky check is told apart from a real regression; regressions and flaky checks recorded as findings; and a new baseline accepted only by a person.", "dependencies": [ "findings-to-fixes", "video-qa", "release-and-verify", "screenshot-design-review", "checks-of-checks" ], "boundaries": [ "Measurements are read-only: page loads and GET requests to a local build or a test target a person named, and local files.", "Only a person accepts a baseline; comparisons never move one, and every earlier set is kept.", "No worker switches or reverts a live version; that stays the release owner's act in the release and verify pack.", "Workers create nouns and propose verbs; an authorized requester enqueues the proposals worth doing.", "Importing the pack writes nothing and enqueues nothing." ], "nouns": [ { "slug": "pack", "category": "noun.skill_pack", "role": "pack", "title": "A collection of nouns and verbs for watching for regressions against baselines a person accepted", "value": { "lets_workers": "Keep baselines as nouns, from page screenshots by hash to metrics with tolerances, API response shapes and video QA numbers; measure every change again under the same settings; compare by fixed rules and measure every mismatch twice more before reporting it, so a flaky check is told apart from a real regression; record regressions as findings and flaky checks as findings against the check; and leave every new baseline to a person.", "flow": [ { "verb": { "$verb": "regression-watch.capture-baseline" }, "when": "Once for the version everyone agrees is right." }, { "verb": { "$verb": "regression-watch.accept-baseline" }, "when": "A person accepts the candidate, or later replaces items or changes a tolerance." }, { "verb": { "$verb": "regression-watch.apply-acceptance" }, "when": "The person's decision written as the next baseline set, exactly." }, { "verb": { "$verb": "regression-watch.remeasure" }, "when": "On every change that could move a baseline." }, { "verb": { "$verb": "release-and-verify.run-smoke" }, "when": "After a release, read-only checks of the live version, compared here as check results." }, { "verb": { "$verb": "video-qa.compile-qa-report" }, "when": "For a new render, the numbers this pack compares." }, { "verb": { "$verb": "regression-watch.compare-to-baseline" }, "when": "The rules, with two reruns of every mismatch." }, { "verb": { "$verb": "regression-watch.report-regressions" }, "when": "Findings for regressions and flaky checks; proposals for a person." }, { "verb": { "$verb": "regression-watch.review-flaky-checks" }, "when": "Weekly, across comparisons." } ], "guardrails": [ { "$pin": "guardrail.compare-like-with-like" }, { "$pin": "guardrail.rerun-before-reporting" }, { "$pin": "guardrail.person-accepts-baseline" }, { "$pin": "video-qa:guardrail.not-run-is-not-a-pass" }, { "$pin": "video-qa:guardrail.measure-the-exact-bytes" } ], "worked_examples": [ { "$pin": "watch-checks" }, { "$pin": "example.baseline-set" }, { "$pin": "example.measurement" }, { "$pin": "example.comparison" }, { "$pin": "example.acceptance" }, { "$pin": "example.new-baseline-set" }, { "$pin": "example.regression-finding" }, { "$pin": "example.flake-finding" } ], "complements": [ "Video numbers come from the video QA pack's extended report on the exact bytes; this pack compares them with the last accepted render.", "After a release, the release and verify pack's parity check and smoke checks read the live version; this pack keeps their results as check-result baselines, and its workers never switch a live version.", "Screenshots are captured with the screenshot design review pack's matrix, and a changed capture is judged by its design rules before a person sees it.", "Findings go through the findings to fixes pack; a check that keeps flaking goes to the checks of checks pack." ] }, "relations": [ [ "requires_noun", "method" ], [ "requires_noun", "guardrail.compare-like-with-like" ], [ "requires_noun", "guardrail.rerun-before-reporting" ], [ "requires_noun", "guardrail.person-accepts-baseline" ], [ "requires_noun", "baseline-kinds" ], [ "requires_noun", "compare-rules" ], [ "requires_noun", "flake-rules" ], [ "requires_noun", "watch-checks" ] ] }, { "slug": "method", "category": "noun.skill_pack", "role": "method", "title": "How baselines are captured, accepted, measured again and compared", "category_reason": "A reusable procedure with guardrails, followed by every verb in this pack.", "value": { "lifecycle": [ "capture-baseline measures the accepted version under named settings and writes a candidate baseline set.", "accept-baseline is a person's verb: they accept a candidate, or later replace items with new measurements or change a tolerance, and every earlier set is kept.", "apply-acceptance writes exactly what the person decided as the next baseline set, and nothing else.", "remeasure measures a change under exactly the baseline's settings.", "compare-to-baseline applies the comparison rules and measures every mismatch again at least twice before it classifies anything.", "report-regressions records each regression as a finding, each flaky check as a finding against the check, and proposes accept-baseline for intended changes, improvements and new items.", "review-flaky-checks looks across comparisons for checks that keep flaking and proposes fixing or retiring them." ], "rules": [ "Compare like with like: the same cell, environment, tool versions and inputs, or the item is not comparable.", "Rerun before reporting: a mismatch is a regression only if every rerun repeats it.", "A missing measurement is missing, never a match; a check that did not run is not a pass.", "Only a person moves a baseline; an improvement is a proposal for them, never an update.", "Every regression becomes a finding with reproduce, propose-fix, fix, verify-fix and add-regression-guard; the watch itself is the guard once the fix lands.", "Workers create nouns and propose verbs; they never enqueue." ], "when": "On every change that could move a baseline: a merge, a build, a render or a release. After a release, the release and verify pack's smoke checks and parity read the live version; this pack compares their results with the baseline." } }, { "slug": "guardrail.compare-like-with-like", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: compare only like with like", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "A measurement is compared only with a baseline taken under the same settings: the same capture cell, environment, tool versions and inputs. Items measured under other settings are not comparable, and are never reported as a regression or as a match.", "how_it_is_checked": "Every comparison states whether the settings matched, and an item whose settings differ is listed as not comparable." } }, { "slug": "guardrail.rerun-before-reporting", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: measure every mismatch again before reporting it", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "Every mismatch is measured again at least twice under the same settings before anything is reported. Only a mismatch every rerun repeats is a regression; one that comes and goes is a flaky check, recorded against the check and not the product.", "how_it_is_checked": "Every regression and every flaky item lists its first run and two reruns." } }, { "slug": "guardrail.person-accepts-baseline", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: only a person moves a baseline", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "A baseline changes only when a person accepts a new one. An improvement, an intended change or a new item never moves a baseline by itself, and every earlier baseline set is kept.", "how_it_is_checked": "Every comparison says the baseline was not updated, and every accepted set names a person's acceptance and the set it supersedes." } }, { "slug": "baseline-kinds", "category": "noun.skill_pack", "role": "baseline_kinds", "title": "Five kinds of baseline and what each stores", "category_reason": "A reusable description of what each kind of baseline holds.", "value": { "kinds": [ { "kind": "screenshot", "stores": "The SHA-256 of a full-page capture and of its layout facts, per capture cell.", "measure_with": "The screenshot design review pack's capture matrix and layout facts, with animations off and fonts settled.", "volatile": "Dates, counters and anything the page says changes; mask them in the capture plan, never after the fact." }, { "kind": "metric", "stores": "A number with its unit, how it was sampled, a tolerance and which direction is better.", "measure_with": "The same tool, version and sample count each time, such as the median of five lab runs.", "volatile": "Network and machine load; that is what reruns and tolerances are for." }, { "kind": "api_shape", "stores": "Every path in a response with its type, and the enum values seen, never the values themselves.", "measure_with": "Read-only requests with synthetic data, against a local build or a test target.", "volatile": "Ids, timestamps and ordering of keys; shapes ignore them." }, { "kind": "video_qa", "stores": "The video QA pack's measured numbers for one file: loudness, true peak, duration, caption cues, blank and frozen frames.", "measure_with": "The video QA pack's measurement rules on the exact delivered bytes.", "volatile": "None: the same bytes give the same numbers." }, { "kind": "check_result", "stores": "A deterministic check's pass or fail, such as a smoke check or a design rule.", "measure_with": "The check's own pinned command.", "volatile": "None; a check that changes its result on the same input is flaky." } ] } }, { "slug": "compare-rules", "category": "noun.reflex_procedure", "role": "compare_rules", "title": "Comparing a measurement with its baseline", "category_reason": "A zero-call rule: the same baseline, measurement and reruns always give the same outcomes.", "value": { "outcomes": { "match": "Within the rule for its kind.", "regression": "Outside the rule in the worse direction, on the first run and on every rerun.", "flaky": "Outside the rule on some runs and within it on others.", "changed": "Outside the rule in a better or neutral direction, or a new item; a person decides whether it becomes the baseline.", "missing": "Not measured this time, with the reason; never a match.", "not_comparable": "Measured under different settings." }, "by_kind": { "screenshot": "Equal digests match. A different digest is changed when the new capture passes the screenshot design review's rules, and a regression when a rule fails.", "metric": "relative: within amount times the baseline; absolute: within amount; exact: equal; at_most or at_least: a ceiling or floor. Outside it, the better direction is changed and the worse one a regression; when no direction is better, any move outside the rule is a regression.", "api_shape": "A removed path, a changed type or a removed enum value is a regression; an added path or enum value is changed.", "video_qa": "Each number by its metric rule; a QA check that passed and now fails is a regression.", "check_result": "pass to fail is a regression; fail to pass is changed. Count failures, never passes: a new passing test can hide a new failure in a count of passes." }, "rerun": "Every outcome other than match is measured again before it is final; see the flake rules.", "model_calls": 0 } }, { "slug": "flake-rules", "category": "noun.reflex_procedure", "role": "flake_rules", "title": "Telling a flaky check from a regression", "category_reason": "A zero-call rule: the same runs always give the same verdict.", "value": { "reruns": { "default": 2, "minimum": 2, "maximum": 5, "spacing": "The first rerun at once, the second after at least a minute, same settings." }, "verdicts": { "regression": "The first run and every rerun are outside the rule in the worse direction.", "changed": "Every run is outside the rule in a better or neutral direction.", "flaky": "Anything else: at least one run within the rule, or runs that move in different directions." }, "after": [ "A flaky item becomes a finding of kind issue against the check, fingerprinted issue//flaky-, with all three runs.", "A check that flakes three times in thirty days goes to the checks of checks pack's validate-checks, or its tolerance to a person.", "To tell a flaky check from a flaky product, run the check three times on the baseline build: steady there means the change behaves intermittently, and the finding becomes an error against the product.", "A flaky item is never reported as a product regression, and never silently dropped." ], "model_calls": 0 } }, { "slug": "watch-checks", "category": "noun.benchmark_suite", "role": "record_checks", "title": "Record checks for baseline comparisons, with synthetic golden comparisons", "category_reason": "Deterministic checks with synthetic known-good and known-bad records; the validator recomputes every verdict.", "value": { "fixture_kind": "finding_checks", "purpose": "Checks every comparison must pass before its regressions are reported: the pinned and accepted baseline, the change and the pinned measurement, settings stated and unlike settings never compared, at least two reruns, every regression rerun and proposed as a finding, every flaky item kept against its check, matches with values and none from checks that did not run, missing items with reasons, every change proposed to a person, and a baseline the comparison never moves. The two synthetic good comparisons pass; each synthetic bad one fails exactly the one check named in fails.", "deterministic_checks": [ { "id": "baseline-pinned", "rule": { "rule": "pattern_present", "path": "baseline_set_ref.value_sha256", "pattern": "^[0-9a-f]{64}$" }, "why": "The comparison pins the exact baseline set." }, { "id": "baseline-accepted", "rule": { "rule": "present", "path": "acceptance_ref" }, "why": "The baseline in force was accepted by a person." }, { "id": "change-named", "rule": { "rule": "present", "path": "change" }, "why": "The change being compared is named." }, { "id": "measurement-pinned", "rule": { "rule": "pattern_present", "path": "measurement_ref.value_sha256", "pattern": "^[0-9a-f]{64}$" }, "why": "The comparison pins the exact measurement." }, { "id": "settings-stated", "rule": { "rule": "enum", "path": "settings_match", "values": [ true, false ] }, "why": "Whether the settings matched is stated." }, { "id": "unlike-settings-not-compared", "rule": { "rule": "when", "if": { "path": "settings_match", "equals": false }, "then": { "rule": "min_items", "path": "not_comparable", "min": 1 } }, "why": "Items measured under other settings are listed as not comparable." }, { "id": "no-matches-when-unlike", "rule": { "rule": "when", "if": { "path": "settings_match", "equals": false }, "then": { "rule": "pattern_absent", "path": "matches", "pattern": "." } }, "why": "Under unlike settings nothing counts as a match." }, { "id": "reruns-enough", "rule": { "rule": "enum", "path": "reruns_per_mismatch", "values": [ 2, 3, 4, 5 ] }, "why": "Every mismatch is measured at least twice more." }, { "id": "regressions-rerun", "rule": { "rule": "each_has", "path": "regressions", "fields": [ "item", "baseline", "first_run", "rerun_1", "rerun_2" ] }, "why": "A regression shows its first run and both reruns." }, { "id": "regressions-recorded", "rule": { "rule": "each_has", "path": "regressions", "fields": [ "item", "finding_fingerprint" ] }, "why": "Every regression is proposed as a finding." }, { "id": "flaky-kept-against-the-check", "rule": { "rule": "each_has", "path": "flaky", "fields": [ "item", "first_run", "rerun_1", "rerun_2", "check_finding_fingerprint" ] }, "why": "A flaky item keeps its runs and becomes a finding against the check." }, { "id": "matches-measured", "rule": { "rule": "each_has", "path": "matches", "fields": [ "item", "value" ] }, "why": "A match names the value measured." }, { "id": "no-not-run-matches", "rule": { "rule": "pattern_absent", "path": "matches", "pattern": "(?i)\\b(not run|not_run|skipped|no value|missing)\\b" }, "why": "Nothing that was not measured is counted as a match." }, { "id": "missing-explained", "rule": { "rule": "each_has", "path": "missing", "fields": [ "item", "reason" ] }, "why": "Every missing item says why." }, { "id": "changes-go-to-a-person", "rule": { "rule": "each_has", "path": "changed", "fields": [ "item", "direction", "proposed_verb" ] }, "why": "Every change, improvement or new item is proposed to a person." }, { "id": "baseline-not-moved", "rule": { "rule": "enum", "path": "baseline_updated", "values": [ false ] }, "why": "A comparison never moves the baseline." } ], "golden_fixtures": [ { "id": "good-with-every-outcome", "label": "known_good", "provenance": "synthetic", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "good-unlike-settings", "label": "known_good", "provenance": "synthetic", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": false, "reruns_per_mismatch": 2, "matches": [], "regressions": [], "flaky": [], "changed": [], "missing": [], "not_comparable": [ { "item": "LCP p75", "reason": "Measured on another machine profile." } ], "baseline_updated": false, "counts": { "match": 0, "regression": 0, "flaky": 0, "changed": 0, "missing": 0, "not_comparable": 1 } } }, { "id": "bad-baseline-unpinned", "label": "known_bad", "fails": "baseline-pinned", "why": "No digest for the baseline.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "latest" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-baseline-unaccepted", "label": "known_bad", "fails": "baseline-accepted", "why": "A baseline nobody accepted.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-no-change", "label": "known_bad", "fails": "change-named", "why": "No change named.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-measurement-unpinned", "label": "known_bad", "fails": "measurement-pinned", "why": "No digest for the measurement.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "today" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-settings-unstated", "label": "known_bad", "fails": "settings-stated", "why": "Settings never compared.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-unlike-compared", "label": "known_bad", "fails": "unlike-settings-not-compared", "why": "Different settings, yet nothing listed as not comparable.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": false, "reruns_per_mismatch": 2, "matches": [], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-match-under-unlike", "label": "known_bad", "fails": "no-matches-when-unlike", "why": "Settings differed, yet an item was called a match.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": false, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [], "flaky": [], "changed": [], "missing": [], "not_comparable": [ { "item": "LCP p75", "reason": "Measured on another machine profile." } ], "baseline_updated": false, "counts": { "match": 0, "regression": 0, "flaky": 0, "changed": 0, "missing": 0, "not_comparable": 1 } } }, { "id": "bad-one-rerun", "label": "known_bad", "fails": "reruns-enough", "why": "Only one rerun.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 1, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-regression-unrerun", "label": "known_bad", "fails": "regressions-rerun", "why": "A regression reported on its first run.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-regression-unrecorded", "label": "known_bad", "fails": "regressions-recorded", "why": "A regression with no finding.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-flake-dropped", "label": "known_bad", "fails": "flaky-kept-against-the-check", "why": "A flaky item with no finding against the check.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-match-unmeasured", "label": "known_bad", "fails": "matches-measured", "why": "A match with no value.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-not-run-match", "label": "known_bad", "fails": "no-not-run-matches", "why": "A check that did not run counted as a match.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "not run, assumed 0" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-missing-silent", "label": "known_bad", "fails": "missing-explained", "why": "A missing item with no reason.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak" } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-change-auto", "label": "known_bad", "fails": "changes-go-to-a-person", "why": "An improvement with no one asked.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } }, { "id": "bad-baseline-moved", "label": "known_bad", "fails": "baseline-not-moved", "why": "The comparison moved the baseline itself.", "finding": { "comparison_id": "RW-90", "baseline_set_ref": { "store": "s.o", "key": "baseline.t-3", "version": 1, "value_sha256": "1111111111111111111111111111111111111111111111111111111111111111" }, "acceptance_ref": "acceptance.t-3", "change": "build 10", "measurement_ref": { "store": "s.o", "key": "measure.t-7", "version": 1, "value_sha256": "2222222222222222222222222222222222222222222222222222222222222222" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "failing tests", "value": "0 tests" } ], "regressions": [ { "item": "GET /shifts shape", "baseline": "ends_at string", "first_run": "ends_at gone", "rerun_1": "ends_at gone", "rerun_2": "ends_at gone", "finding_fingerprint": "error/api/shape-ends-at-removed" } ], "flaky": [ { "item": "LCP p75", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-check/flaky-lcp-p75" } ], "changed": [ { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "true peak", "reason": "The sound check did not run." } ], "not_comparable": [], "baseline_updated": true, "counts": { "match": 1, "regression": 1, "flaky": 1, "changed": 1, "missing": 1, "not_comparable": 0 } } } ], "expected": { "good-with-every-outcome": { "verdict": "pass", "failed_checks": [] }, "good-unlike-settings": { "verdict": "pass", "failed_checks": [] }, "bad-baseline-unpinned": { "verdict": "fail", "failed_checks": [ "baseline-pinned" ] }, "bad-baseline-unaccepted": { "verdict": "fail", "failed_checks": [ "baseline-accepted" ] }, "bad-no-change": { "verdict": "fail", "failed_checks": [ "change-named" ] }, "bad-measurement-unpinned": { "verdict": "fail", "failed_checks": [ "measurement-pinned" ] }, "bad-settings-unstated": { "verdict": "fail", "failed_checks": [ "settings-stated" ] }, "bad-unlike-compared": { "verdict": "fail", "failed_checks": [ "unlike-settings-not-compared" ] }, "bad-match-under-unlike": { "verdict": "fail", "failed_checks": [ "no-matches-when-unlike" ] }, "bad-one-rerun": { "verdict": "fail", "failed_checks": [ "reruns-enough" ] }, "bad-regression-unrerun": { "verdict": "fail", "failed_checks": [ "regressions-rerun" ] }, "bad-regression-unrecorded": { "verdict": "fail", "failed_checks": [ "regressions-recorded" ] }, "bad-flake-dropped": { "verdict": "fail", "failed_checks": [ "flaky-kept-against-the-check" ] }, "bad-match-unmeasured": { "verdict": "fail", "failed_checks": [ "matches-measured" ] }, "bad-not-run-match": { "verdict": "fail", "failed_checks": [ "no-not-run-matches" ] }, "bad-missing-silent": { "verdict": "fail", "failed_checks": [ "missing-explained" ] }, "bad-change-auto": { "verdict": "fail", "failed_checks": [ "changes-go-to-a-person" ] }, "bad-baseline-moved": { "verdict": "fail", "failed_checks": [ "baseline-not-moved" ] } } } }, { "slug": "template.baseline", "category": "noun.eval_run", "role": "template", "title": "Template: a baseline set: the subject, the change it was measured on, the settings, every item with its value, unit and tolerance, and whether a person accepted it", "category_reason": "A baseline is a recorded measurement run that later runs are compared with.", "value": { "describes": "a baseline set: the subject, the change it was measured on, the settings, every item with its value, unit and tolerance, and whether a person accepted it", "schema": { "type": "object", "additionalProperties": false, "required": [ "baseline_set_id", "subject", "change", "settings", "captured_at", "items", "status", "acceptance_ref", "supersedes" ], "properties": { "baseline_set_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "subject": { "type": "string", "minLength": 1, "maxLength": 200 }, "change": { "type": "string", "minLength": 1, "maxLength": 200 }, "settings": { "type": "object", "additionalProperties": false, "required": [ "environment", "tools", "inputs" ], "properties": { "environment": { "type": "string", "minLength": 1, "maxLength": 200 }, "tools": { "type": "string", "minLength": 1, "maxLength": 200 }, "inputs": { "type": "string", "minLength": 1, "maxLength": 200 } } }, "captured_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "items": { "type": "array", "minItems": 1, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "kind", "value", "unit", "tolerance" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "kind": { "type": "string", "enum": [ "screenshot", "metric", "api_shape", "video_qa", "check_result" ] }, "value": { "type": [ "string", "number" ] }, "unit": { "type": "string", "minLength": 0, "maxLength": 20 }, "tolerance": { "type": "object", "additionalProperties": false, "required": [ "rule", "amount", "better" ], "properties": { "rule": { "type": "string", "enum": [ "digest", "relative", "absolute", "exact", "at_most", "at_least", "shape" ] }, "amount": { "type": [ "number", "null" ], "minimum": -1000000, "maximum": 1000000 }, "better": { "type": "string", "enum": [ "lower", "higher", "none" ] } } } } } }, "status": { "type": "string", "enum": [ "candidate", "accepted" ] }, "acceptance_ref": { "type": [ "string", "null" ], "maxLength": 200 }, "supersedes": { "type": [ "string", "null" ], "maxLength": 200 } } }, "rules": [ "status is accepted only with an acceptance_ref to a person's acceptance.", "An accepted set is never edited; a new set supersedes it." ] } }, { "slug": "template.measurement", "category": "noun.eval_run", "role": "template", "title": "Template: a measurement of one change under a baseline set's settings, with every item not measured and why", "category_reason": "A measurement is a recorded run.", "value": { "describes": "a measurement of one change under a baseline set's settings, with every item not measured and why", "schema": { "type": "object", "additionalProperties": false, "required": [ "measurement_id", "baseline_set_ref", "change", "settings", "measured_at", "items", "not_measured" ], "properties": { "measurement_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "baseline_set_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "change": { "type": "string", "minLength": 1, "maxLength": 200 }, "settings": { "type": "object", "additionalProperties": false, "required": [ "environment", "tools", "inputs" ], "properties": { "environment": { "type": "string", "minLength": 1, "maxLength": 200 }, "tools": { "type": "string", "minLength": 1, "maxLength": 200 }, "inputs": { "type": "string", "minLength": 1, "maxLength": 200 } } }, "measured_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "items": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "value" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "value": { "type": [ "string", "number" ] } } } }, "not_measured": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "reason" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 } } } } } }, "rules": [ "settings equal the baseline set's, or the comparison lists the items as not comparable." ] } }, { "slug": "template.comparison", "category": "noun.eval_run", "role": "template", "title": "Template: a comparison of a measurement with its accepted baseline, every mismatch measured twice more, and each item's outcome", "category_reason": "A comparison is a recorded verification run.", "value": { "describes": "a comparison of a measurement with its accepted baseline, every mismatch measured twice more, and each item's outcome", "schema": { "type": "object", "additionalProperties": false, "required": [ "comparison_id", "baseline_set_ref", "acceptance_ref", "change", "measurement_ref", "settings_match", "reruns_per_mismatch", "matches", "regressions", "flaky", "changed", "missing", "not_comparable", "baseline_updated", "counts" ], "properties": { "comparison_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "baseline_set_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "acceptance_ref": { "type": "string", "minLength": 1, "maxLength": 200 }, "change": { "type": "string", "minLength": 1, "maxLength": 200 }, "measurement_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "settings_match": { "type": "boolean" }, "reruns_per_mismatch": { "type": "integer", "minimum": 2, "maximum": 5 }, "matches": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "value" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "value": { "type": "string", "minLength": 1, "maxLength": 200 } } } }, "regressions": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "baseline", "first_run", "rerun_1", "rerun_2", "finding_fingerprint" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "baseline": { "type": "string", "minLength": 1, "maxLength": 200 }, "first_run": { "type": "string", "minLength": 1, "maxLength": 200 }, "rerun_1": { "type": "string", "minLength": 1, "maxLength": 200 }, "rerun_2": { "type": "string", "minLength": 1, "maxLength": 200 }, "more_reruns": { "type": "array", "minItems": 0, "maxItems": 3, "items": { "type": "string", "minLength": 1, "maxLength": 200 } }, "finding_fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 } } } }, "flaky": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "baseline", "first_run", "rerun_1", "rerun_2", "check_finding_fingerprint" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "baseline": { "type": "string", "minLength": 1, "maxLength": 200 }, "first_run": { "type": "string", "minLength": 1, "maxLength": 200 }, "rerun_1": { "type": "string", "minLength": 1, "maxLength": 200 }, "rerun_2": { "type": "string", "minLength": 1, "maxLength": 200 }, "more_reruns": { "type": "array", "minItems": 0, "maxItems": 3, "items": { "type": "string", "minLength": 1, "maxLength": 200 } }, "check_finding_fingerprint": { "type": "string", "minLength": 1, "maxLength": 160 } } } }, "changed": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "direction", "first_run", "rerun_1", "rerun_2", "proposed_verb" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "direction": { "type": "string", "enum": [ "better", "neutral", "new" ] }, "first_run": { "type": "string", "minLength": 1, "maxLength": 200 }, "rerun_1": { "type": "string", "minLength": 1, "maxLength": 200 }, "rerun_2": { "type": "string", "minLength": 1, "maxLength": 200 }, "more_reruns": { "type": "array", "minItems": 0, "maxItems": 3, "items": { "type": "string", "minLength": 1, "maxLength": 200 } }, "proposed_verb": { "type": "string", "minLength": 1, "maxLength": 120 } } } }, "missing": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "reason" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "not_comparable": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "reason" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "baseline_updated": { "type": "boolean", "enum": [ false ] }, "counts": { "type": "object", "additionalProperties": false, "required": [ "match", "regression", "flaky", "changed", "missing", "not_comparable" ], "properties": { "match": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "regression": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "flaky": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "changed": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "missing": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "not_comparable": { "type": "integer", "minimum": 0, "maximum": 1000000 } } } } }, "rules": [ "Passes watch-checks.", "counts equal the length of each list." ] } }, { "slug": "template.baseline-acceptance", "category": "noun.eval_run", "role": "template", "title": "Template: a person's decision on which items become the new baseline and which keep the old", "category_reason": "A person's acceptance is a recorded decision.", "value": { "describes": "a person's decision on which items become the new baseline and which keep the old", "schema": { "type": "object", "additionalProperties": false, "required": [ "acceptance_id", "baseline_set_ref", "comparison_ref", "decisions", "new_baseline_set", "keeps", "decided_by_role", "decided_by_kind", "decided_at" ], "properties": { "acceptance_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "baseline_set_ref": { "type": "object", "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "comparison_ref": { "type": [ "object", "null" ], "additionalProperties": false, "required": [ "store", "key", "version", "value_sha256" ], "properties": { "store": { "type": "string", "minLength": 1, "maxLength": 64 }, "key": { "type": "string", "minLength": 1, "maxLength": 256 }, "version": { "type": "integer", "minimum": 1, "maximum": 1000000000 }, "value_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "decisions": { "type": "array", "minItems": 1, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "decision", "reason" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 120 }, "decision": { "type": "string", "enum": [ "accept_new", "keep_old", "change_tolerance" ] }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 }, "tolerance": { "type": [ "object", "null" ], "additionalProperties": false, "required": [ "rule", "amount", "better" ], "properties": { "rule": { "type": "string", "enum": [ "digest", "relative", "absolute", "exact", "at_most", "at_least", "shape" ] }, "amount": { "type": [ "number", "null" ], "minimum": -1000000, "maximum": 1000000 }, "better": { "type": "string", "enum": [ "lower", "higher", "none" ] } } } } } }, "new_baseline_set": { "type": [ "string", "null" ], "maxLength": 120 }, "keeps": { "type": "string", "minLength": 1, "maxLength": 200 }, "decided_by_role": { "type": "string", "minLength": 1, "maxLength": 120 }, "decided_by_kind": { "type": "string", "enum": [ "person" ] }, "decided_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 } } }, "rules": [ "Recorded by a person; workers never write it.", "The old set is kept and named." ] } }, { "slug": "template.flake-review", "category": "noun.eval_run", "role": "template", "title": "Template: a review of which checks flaked across comparisons in a window, with a proposal for each", "category_reason": "A flake review is a recorded analysis run.", "value": { "describes": "a review of which checks flaked across comparisons in a window, with a proposal for each", "schema": { "type": "object", "additionalProperties": false, "required": [ "review_id", "window", "comparisons", "checks" ], "properties": { "review_id": { "type": "string", "minLength": 1, "maxLength": 60, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,40}$" }, "window": { "type": "object", "additionalProperties": false, "required": [ "from", "to" ], "properties": { "from": { "type": "string", "minLength": 10, "maxLength": 10 }, "to": { "type": "string", "minLength": 10, "maxLength": 10 } } }, "comparisons": { "type": "integer", "minimum": 1, "maximum": 10000 }, "checks": { "type": "array", "minItems": 0, "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "check", "flakes", "runs", "proposal" ], "properties": { "check": { "type": "string", "minLength": 1, "maxLength": 120 }, "flakes": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "runs": { "type": "integer", "minimum": 0, "maximum": 1000000 }, "proposal": { "type": "string", "enum": [ "validate_check", "widen_tolerance_with_a_person", "fix_the_environment", "keep_watching" ] } } } } } }, "rules": [ "A widened tolerance is a baseline change, so it goes to a person." ] } }, { "slug": "test-cases", "category": "noun.benchmark_suite", "role": "test_cases", "title": "Synthetic test cases for regression watch", "synthetic": true, "value": { "cases": [ { "id": "RW-FIRST-RUN-ONLY", "scenario": "A worker reports a slower page from one run", "sample_input": "SYNTHETIC: LCP 2480 ms once against 2100 ms.", "expected_outcome": "change_compared", "expected_indicators": [ "reruns_taken", "flaky_or_regression_decided_after" ], "expected_human_escalation": false }, { "id": "RW-OTHER-MACHINE", "scenario": "The measurement ran on a different machine profile", "sample_input": "SYNTHETIC: settings differ from the baseline's.", "expected_outcome": "change_compared", "expected_indicators": [ "not_comparable", "no_regression_reported" ], "expected_human_escalation": false }, { "id": "RW-IMPROVEMENT", "scenario": "The bundle got smaller", "sample_input": "SYNTHETIC: 182 KB to 176 KB.", "expected_outcome": "change_compared", "expected_indicators": [ "changed_better", "accept_baseline_proposed", "baseline_not_updated" ], "expected_human_escalation": true }, { "id": "RW-CHECK-NOT-RUN", "scenario": "A video QA check did not run", "sample_input": "SYNTHETIC: no true peak this render.", "expected_outcome": "change_compared", "expected_indicators": [ "missing_with_reason", "not_a_match" ], "expected_human_escalation": false }, { "id": "RW-NO-ACCEPTED-BASELINE", "scenario": "Only a candidate baseline exists", "sample_input": "SYNTHETIC: BS-1 is a candidate.", "expected_outcome": "blocked", "expected_indicators": [ "accept_baseline_needed" ], "expected_human_escalation": true }, { "id": "RW-WORKER-ACCEPTS", "scenario": "A worker tries to accept a new baseline", "sample_input": "SYNTHETIC: an agent key writes an acceptance.", "expected_outcome": "blocked", "expected_indicators": [ "person_only" ], "expected_human_escalation": true } ] } }, { "slug": "example.baseline-set", "category": "noun.eval_run", "role": "example", "title": "Worked example: an accepted baseline set for a site, its API and its video", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.baseline" }, "instance": { "baseline_set_id": "BS-3", "subject": "SYNTHETIC pantry site, its shifts API and its 45-second explainer", "change": "site build 2026-10-20.1", "settings": { "environment": "Chromium 140 on the pinned lab machine profile; local build with synthetic data", "tools": "capture plan CP-2, lab runner 5 samples, video QA code at its pinned digest", "inputs": "synthetic pantry data set D-4; explainer file by digest" }, "captured_at": "2026-10-20T15:00Z", "items": [ { "item": "screenshot /shifts 390-light", "kind": "screenshot", "value": "3333333333333333333333333333333333333333333333333333333333333333", "unit": "", "tolerance": { "rule": "digest", "amount": null, "better": "none" } }, { "item": "screenshot /shifts 1440-dark", "kind": "screenshot", "value": "4444444444444444444444444444444444444444444444444444444444444444", "unit": "", "tolerance": { "rule": "digest", "amount": null, "better": "none" } }, { "item": "LCP p75, five lab runs", "kind": "metric", "value": 2100, "unit": "ms", "tolerance": { "rule": "relative", "amount": 0.1, "better": "lower" } }, { "item": "JS bundle", "kind": "metric", "value": 182, "unit": "KB", "tolerance": { "rule": "relative", "amount": 0.02, "better": "lower" } }, { "item": "failing tests", "kind": "check_result", "value": 0, "unit": "tests", "tolerance": { "rule": "exact", "amount": null, "better": "lower" } }, { "item": "GET /api/shifts shape", "kind": "api_shape", "value": "next:string|null; shifts:array; shifts[].ends_at:string; shifts[].id:string; shifts[].spots_left:integer; shifts[].starts_at:string", "unit": "", "tolerance": { "rule": "shape", "amount": null, "better": "none" } }, { "item": "video integrated loudness", "kind": "video_qa", "value": -16.0, "unit": "LUFS", "tolerance": { "rule": "absolute", "amount": 1.0, "better": "none" } }, { "item": "video true peak", "kind": "video_qa", "value": -1.4, "unit": "dBTP", "tolerance": { "rule": "at_most", "amount": -1.0, "better": "lower" } }, { "item": "video caption cues", "kind": "video_qa", "value": 14, "unit": "cues", "tolerance": { "rule": "exact", "amount": null, "better": "none" } }, { "item": "video duration", "kind": "video_qa", "value": 45.0, "unit": "s", "tolerance": { "rule": "absolute", "amount": 0.04, "better": "none" } } ], "status": "accepted", "acceptance_ref": "acceptance AC-3 by the site owner", "supersedes": "BS-2" }, "shows": "Ten items of all five kinds, each with its tolerance, accepted by a person and superseding the set before.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.measurement", "category": "noun.eval_run", "role": "example", "title": "Worked example: one change measured under the baseline's settings", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.measurement" }, "instance": { "measurement_id": "M-7", "baseline_set_ref": { "$pin": "example.baseline-set" }, "change": "site build 2026-10-30.5 (commit 4f2a9c1)", "settings": { "environment": "Chromium 140 on the pinned lab machine profile; local build with synthetic data", "tools": "capture plan CP-2, lab runner 5 samples, video QA code at its pinned digest", "inputs": "synthetic pantry data set D-4; explainer file by digest" }, "measured_at": "2026-10-30T14:00Z", "items": [ { "item": "screenshot /shifts 390-light", "value": "5555555555555555555555555555555555555555555555555555555555555555" }, { "item": "screenshot /shifts 1440-dark", "value": "4444444444444444444444444444444444444444444444444444444444444444" }, { "item": "LCP p75, five lab runs", "value": 2480 }, { "item": "JS bundle", "value": 176 }, { "item": "failing tests", "value": 0 }, { "item": "GET /api/shifts shape", "value": "next:string|null; shifts:array; shifts[].id:string; shifts[].spots_left:integer; shifts[].starts_at:string" }, { "item": "video integrated loudness", "value": -16.2 }, { "item": "video caption cues", "value": 13 }, { "item": "video duration", "value": 45.0 } ], "not_measured": [ { "item": "video true peak", "reason": "The video QA pack's check-sound did not run on this render." } ] }, "shows": "Nine items measured and one not measured, with the reason.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.comparison", "category": "noun.eval_run", "role": "example", "title": "Worked example: every outcome, after reruns", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.comparison" }, "instance": { "comparison_id": "RW-7", "baseline_set_ref": { "$pin": "example.baseline-set" }, "acceptance_ref": "acceptance AC-3 by the site owner", "change": "site build 2026-10-30.5 (commit 4f2a9c1)", "measurement_ref": { "$pin": "example.measurement" }, "settings_match": true, "reruns_per_mismatch": 2, "matches": [ { "item": "screenshot /shifts 1440-dark", "value": "sha256 444444444444" }, { "item": "failing tests", "value": "0 tests" }, { "item": "video integrated loudness", "value": "-16.2 LUFS" }, { "item": "video duration", "value": "45 s" } ], "regressions": [ { "item": "GET /api/shifts shape", "baseline": "shifts[].ends_at:string present", "first_run": "shifts[].ends_at removed", "rerun_1": "shifts[].ends_at removed", "rerun_2": "shifts[].ends_at removed", "finding_fingerprint": "error/shifts-api/shape-ends-at-removed" }, { "item": "video caption cues", "baseline": "14 cues", "first_run": "13 cues", "rerun_1": "13 cues", "rerun_2": "13 cues", "finding_fingerprint": "error/explainer-video/caption-cue-lost" } ], "flaky": [ { "item": "LCP p75, five lab runs", "baseline": "2100 ms", "first_run": "2480 ms", "rerun_1": "2150 ms", "rerun_2": "2090 ms", "check_finding_fingerprint": "issue/lcp-lab-check/flaky-lcp-p75" } ], "changed": [ { "item": "screenshot /shifts 390-light", "direction": "neutral", "first_run": "sha256 555555555555", "rerun_1": "sha256 555555555555", "rerun_2": "sha256 555555555555", "proposed_verb": "regression-watch.accept-baseline" }, { "item": "JS bundle", "direction": "better", "first_run": "176 KB", "rerun_1": "176 KB", "rerun_2": "176 KB", "proposed_verb": "regression-watch.accept-baseline" } ], "missing": [ { "item": "video true peak", "reason": "The video QA pack's check-sound did not run on this render." } ], "not_comparable": [], "baseline_updated": false, "counts": { "match": 4, "regression": 2, "flaky": 1, "changed": 2, "missing": 1, "not_comparable": 0 } }, "shows": "Four matches, a removed API field and a lost caption cue confirmed by two reruns each, a lab metric that came back on rerun and is flaky, an intended screenshot change and a smaller bundle for a person, and a true peak that was not measured.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.acceptance", "category": "noun.eval_run", "role": "example", "title": "Worked example: a person accepts two items, widens one tolerance and keeps the rest", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.baseline-acceptance" }, "instance": { "acceptance_id": "AC-4", "baseline_set_ref": { "$pin": "example.baseline-set" }, "comparison_ref": { "$pin": "example.comparison" }, "decisions": [ { "item": "screenshot /shifts 390-light", "decision": "accept_new", "reason": "The new add-to-calendar button is intended." }, { "item": "JS bundle", "decision": "accept_new", "reason": "The smaller bundle is the new bar." }, { "item": "LCP p75, five lab runs", "decision": "change_tolerance", "tolerance": { "rule": "relative", "amount": 0.15, "better": "lower" }, "reason": "Widen to 15 percent while the lab runner's noise is fixed; the 18 percent run stays recorded as flaky." }, { "item": "GET /api/shifts shape", "decision": "keep_old", "reason": "Clients still read ends_at; the removal is the regression." }, { "item": "video caption cues", "decision": "keep_old", "reason": "A cue was lost at the last cut; fix the render." } ], "new_baseline_set": "BS-4: BS-3 with the two accepted items' measured values and the LCP tolerance changed", "keeps": "BS-3, superseded and kept", "decided_by_role": "site owner", "decided_by_kind": "person", "decided_at": "2026-10-30T16:10Z" }, "shows": "Two items replaced, one tolerance widened, two regressions refused as baselines, the old set kept.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.new-baseline-set", "category": "noun.eval_run", "role": "example", "title": "Worked example: the next baseline set, written exactly as the person decided", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.baseline" }, "instance": { "baseline_set_id": "BS-4", "subject": "SYNTHETIC pantry site, its shifts API and its 45-second explainer", "change": "site build 2026-10-30.5 (commit 4f2a9c1)", "settings": { "environment": "Chromium 140 on the pinned lab machine profile; local build with synthetic data", "tools": "capture plan CP-2, lab runner 5 samples, video QA code at its pinned digest", "inputs": "synthetic pantry data set D-4; explainer file by digest" }, "captured_at": "2026-10-30T16:20Z", "items": [ { "item": "screenshot /shifts 390-light", "kind": "screenshot", "value": "5555555555555555555555555555555555555555555555555555555555555555", "unit": "", "tolerance": { "rule": "digest", "amount": null, "better": "none" } }, { "item": "screenshot /shifts 1440-dark", "kind": "screenshot", "value": "4444444444444444444444444444444444444444444444444444444444444444", "unit": "", "tolerance": { "rule": "digest", "amount": null, "better": "none" } }, { "item": "LCP p75, five lab runs", "kind": "metric", "value": 2100, "unit": "ms", "tolerance": { "rule": "relative", "amount": 0.15, "better": "lower" } }, { "item": "JS bundle", "kind": "metric", "value": 176, "unit": "KB", "tolerance": { "rule": "relative", "amount": 0.02, "better": "lower" } }, { "item": "failing tests", "kind": "check_result", "value": 0, "unit": "tests", "tolerance": { "rule": "exact", "amount": null, "better": "lower" } }, { "item": "GET /api/shifts shape", "kind": "api_shape", "value": "next:string|null; shifts:array; shifts[].ends_at:string; shifts[].id:string; shifts[].spots_left:integer; shifts[].starts_at:string", "unit": "", "tolerance": { "rule": "shape", "amount": null, "better": "none" } }, { "item": "video integrated loudness", "kind": "video_qa", "value": -16.0, "unit": "LUFS", "tolerance": { "rule": "absolute", "amount": 1.0, "better": "none" } }, { "item": "video true peak", "kind": "video_qa", "value": -1.4, "unit": "dBTP", "tolerance": { "rule": "at_most", "amount": -1.0, "better": "lower" } }, { "item": "video caption cues", "kind": "video_qa", "value": 14, "unit": "cues", "tolerance": { "rule": "exact", "amount": null, "better": "none" } }, { "item": "video duration", "kind": "video_qa", "value": 45.0, "unit": "s", "tolerance": { "rule": "absolute", "amount": 0.04, "better": "none" } } ], "status": "accepted", "acceptance_ref": "acceptance AC-4 by the site owner", "supersedes": "BS-3" }, "shows": "BS-3 copied with only the two accepted values and the widened tolerance changed, accepted by the person's acceptance and superseding BS-3.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.regression-finding", "category": "noun.seed_hypothesis", "role": "example", "title": "Worked example: the removed API field, in the findings to fixes shape", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "findings-to-fixes:template.finding" }, "instance": { "id": "RG-1", "title": "The shifts API stopped returning ends_at for each shift", "kind": "error", "severity": "P1", "where": "SYNTHETIC pantry shifts API, GET /api/shifts, site build 2026-10-30.5", "expected": "Every shift has ends_at as a string, as in baseline set BS-3", "evidence": [ { "kind": "measurement", "detail": "api shape: shifts[].ends_at:string in the baseline; removed on the first run and both reruns", "location": "comparison RW-7", "sha256": "6666666666666666666666666666666666666666666666666666666666666666" }, { "kind": "reproduction", "detail": "Read GET /api/shifts on the local build at commit 4f2a9c1 and list the keys of the first shift.", "location": "local build with synthetic data set D-4" } ], "noticed_by": "compare-to-baseline worker", "noticed_at": "2026-10-30T14:20Z", "fingerprint": "error/shifts-api/shape-ends-at-removed", "status": "open", "sensitivity": "none", "public": false, "work": [ { "verb": "reproduce", "state": "pending" }, { "verb": "propose-fix", "state": "pending" }, { "verb": "fix", "state": "pending" }, { "verb": "verify-fix", "state": "pending" }, { "verb": "add-regression-guard", "state": "pending" } ] }, "shows": "A regression confirmed by reruns, with reproduce, propose-fix, fix, verify-fix and add-regression-guard proposed.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } }, { "slug": "example.flake-finding", "category": "noun.seed_hypothesis", "role": "example", "title": "Worked example: the flaky lab check, as a finding against the check", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "findings-to-fixes:template.finding" }, "instance": { "id": "RG-2", "title": "The lab LCP check reads 18 percent slower on one run in three on the same build", "kind": "issue", "severity": "P3", "where": "SYNTHETIC lab runner, LCP p75 check, site build 2026-10-30.5", "evidence": [ { "kind": "measurement", "detail": "LCP p75: 2480 ms, then 2150 ms and 2090 ms on reruns, against 2100 ms with a 10 percent tolerance", "location": "comparison RW-7", "sha256": "7777777777777777777777777777777777777777777777777777777777777777" } ], "noticed_by": "compare-to-baseline worker", "noticed_at": "2026-10-30T14:20Z", "fingerprint": "issue/lcp-lab-check/flaky-lcp-p75", "status": "open", "sensitivity": "none", "public": false, "work": [ { "verb": "reproduce", "state": "pending" }, { "verb": "propose-fix", "state": "pending" }, { "verb": "fix", "state": "pending" }, { "verb": "verify-fix", "state": "pending" }, { "verb": "add-regression-guard", "state": "pending" } ] }, "shows": "A flaky check recorded as an issue with all three runs, never as a product regression.", "source": "Synthetic. Written for this pack; every name, address, number, digest and identifier is made up." } } ], "verbs": [ { "slug": "capture-baseline", "kind": "verb.benchmark_harness", "title": "Capture a candidate baseline", "claimant": "agent", "summary": "Measure the version everyone agrees is right under named settings, from screenshots and metrics to API shapes, check results and video QA numbers, and write a candidate baseline set for a person to accept.", "outcome": "A candidate baseline set.", "ready_outcome": "baseline_captured", "pinned": [ "baseline-kinds", "compare-rules", "screenshot-design-review:capture-matrix", "video-qa:checks.extended", "release-and-verify:smoke-checklist" ], "slots": [ { "name": "subject_brief", "category": "noun.seed_hypothesis", "description": "Pin of a brief naming the pages, endpoints, checks and files to baseline, and the settings to use." }, { "name": "qa_report", "category": "noun.eval_run", "description": "Pin of the video QA pack's extended report, when a video is in the set.", "required": false } ], "produces": [ { "name": "baseline", "template": "template.baseline" } ], "steps": [ "Fix the settings first: environment, tool versions, capture plan and inputs, and record them.", "Measure each item with the tool its kind names; mask volatile parts in the capture plan before capturing.", "Give each item its tolerance and better direction from the requester's rules or the compare rules' defaults.", "Write the set as a candidate; never mark it accepted." ], "acceptance": [ { "id": "baseline-matches-template", "check": "The baseline validates against template.baseline's schema.", "method": "deterministic" }, { "id": "candidate-only", "check": "status is candidate and acceptance_ref is null.", "method": "deterministic" }, { "id": "settings-recorded", "check": "Every setting is filled.", "method": "deterministic" } ], "network": [ "read_only", "Read-only page loads and GET requests to the pages and endpoints the baseline names, on a local build or a test target a person named, with synthetic data; video files are read locally." ], "bounds": { "max_elapsed_seconds": 3600 } }, { "slug": "accept-baseline", "kind": "verb.verify_gate", "title": "Accept a baseline (a person)", "claimant": "human", "summary": "A person reads the candidate set, or a comparison's proposed changes, and decides item by item which values become the baseline and which keep the old, keeping every earlier set. This template performs nothing itself.", "outcome": "A person's baseline acceptance.", "ready_outcome": "baseline_decided", "model_use": "none", "pinned": [ "guardrail.person-accepts-baseline" ], "slots": [ { "name": "baseline", "category": "noun.eval_run", "description": "Pin of the baseline set being accepted or replaced." }, { "name": "comparison", "category": "noun.eval_run", "description": "Pin of the comparison whose changes are proposed, if any.", "required": false } ], "produces": [ { "name": "acceptance", "template": "template.baseline-acceptance" } ], "steps": [ "Read each proposed item with its old and new values and the comparison's reruns.", "Accept only what you would want every later change compared against; keep the old value for anything that is a regression.", "Name the new set and the set it supersedes." ], "acceptance": [ { "id": "acceptance-matches-template", "check": "The acceptance validates against template.baseline-acceptance's schema.", "method": "deterministic" }, { "id": "person-decided", "check": "decided_by_kind is person.", "method": "deterministic" }, { "id": "old-set-kept", "check": "keeps names the superseded set.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 86400 } }, { "slug": "apply-acceptance", "kind": "verb.compile_dag", "title": "Write the next baseline set from a person's acceptance", "claimant": "agent", "summary": "Write the next baseline set exactly as the person's acceptance decided: accepted items take their measured values from the comparison, changed tolerances take the person's values, every other item is copied unchanged, and the old set is named as superseded.", "outcome": "The next accepted baseline set.", "ready_outcome": "baseline_applied", "model_use": "none", "pinned": [ "guardrail.person-accepts-baseline", "compare-rules" ], "slots": [ { "name": "baseline", "category": "noun.eval_run", "description": "Pin of the baseline set the acceptance answers." }, { "name": "comparison", "category": "noun.eval_run", "description": "Pin of the comparison whose values are accepted." }, { "name": "acceptance", "category": "noun.eval_run", "description": "Pin of the person's acceptance." } ], "produces": [ { "name": "baseline", "template": "template.baseline" } ], "steps": [ "Read the acceptance, the baseline set it names and the comparison it answers; if the acceptance is not a person's, end blocked.", "Copy every item; replace only the values of items marked accept_new, from the comparison's first run, and the tolerances of items marked change_tolerance.", "Set status accepted, acceptance_ref to the acceptance and supersedes to the old set; never edit the old set.", "Write nothing the acceptance did not decide; a decision you cannot apply ends blocked." ], "acceptance": [ { "id": "baseline-matches-template", "check": "The new set validates against template.baseline's schema.", "method": "deterministic" }, { "id": "only-decided-items-change", "check": "Every item without an accept_new or change_tolerance decision equals the old set's.", "method": "deterministic" }, { "id": "names-its-history", "check": "acceptance_ref names the acceptance and supersedes names the old set.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 600 }, "follows": [ "regression-watch.accept-baseline" ] }, { "slug": "remeasure", "kind": "verb.benchmark_harness", "title": "Measure a change under the baseline's settings", "claimant": "agent", "summary": "Measure every item of the accepted baseline set on a change, under exactly its settings, and list every item not measured with the reason.", "outcome": "A measurement of the change.", "ready_outcome": "change_measured", "pinned": [ "baseline-kinds", "guardrail.compare-like-with-like", "video-qa:guardrail.measure-the-exact-bytes" ], "slots": [ { "name": "baseline", "category": "noun.eval_run", "description": "Pin of the accepted baseline set." }, { "name": "change", "category": "noun.code_node", "description": "Pin of the change: the commit, build or render to measure." }, { "name": "qa_report", "category": "noun.eval_run", "description": "Pin of the new render's extended QA report, when a video is in the set.", "required": false } ], "produces": [ { "name": "measurement", "template": "template.measurement" } ], "steps": [ "Read the accepted set's settings and reproduce them exactly; record any setting you cannot reproduce.", "Measure every item with the same tool and sample count as the baseline.", "List every item not measured with its reason; never copy a value from the baseline." ], "acceptance": [ { "id": "measurement-matches-template", "check": "The measurement validates against template.measurement's schema.", "method": "deterministic" }, { "id": "every-item-accounted", "check": "Every baseline item is in items or not_measured, once.", "method": "deterministic" } ], "network": [ "read_only", "Read-only page loads and GET requests to the pages and endpoints the baseline names, on a local build or a test target a person named, with synthetic data; video files are read locally." ], "bounds": { "max_elapsed_seconds": 3600 }, "follows": [ "regression-watch.accept-baseline" ] }, { "slug": "compare-to-baseline", "kind": "verb.verify_gate", "title": "Compare with the baseline, rerunning every mismatch", "claimant": "agent", "summary": "Apply the comparison rules to every item, measure every item that did not match twice more under the same settings, and classify each as match, regression, flaky, changed, missing or not comparable.", "outcome": "A comparison with its reruns.", "ready_outcome": "change_compared", "model_use": "none", "pinned": [ "compare-rules", "flake-rules", "watch-checks", "guardrail.compare-like-with-like", "guardrail.rerun-before-reporting", "video-qa:guardrail.not-run-is-not-a-pass", "release-and-verify:parity-methods" ], "slots": [ { "name": "baseline", "category": "noun.eval_run", "description": "Pin of the accepted baseline set." }, { "name": "measurement", "category": "noun.eval_run", "description": "Pin of the measurement." }, { "name": "acceptance", "category": "noun.eval_run", "description": "Pin of the person's acceptance of the baseline set." } ], "produces": [ { "name": "comparison", "template": "template.comparison" } ], "steps": [ "Check the settings first; list every item measured under other settings as not comparable.", "Apply each item's rule; for every outcome other than match, measure the item twice more, the second time after at least a minute.", "Classify by the flake rules from all three runs, and list every missing item with its reason.", "Never change the baseline; propose accept-baseline for every change, improvement and new item." ], "acceptance": [ { "id": "comparison-matches-template", "check": "The comparison validates against template.comparison's schema.", "method": "deterministic" }, { "id": "watch-checks-pass", "check": "The comparison passes every pinned watch check.", "method": "deterministic" }, { "id": "outcomes-recompute", "check": "Recomputing the rules from the baseline, the measurement and the reruns gives the same outcomes.", "method": "deterministic" } ], "network": [ "read_only", "Read-only page loads and GET requests to the pages and endpoints the baseline names, on a local build or a test target a person named, with synthetic data; video files are read locally." ], "bounds": { "max_elapsed_seconds": 3600 }, "ready_requires": [ "proposed_verbs" ], "follows": [ "regression-watch.remeasure" ] }, { "slug": "report-regressions", "kind": "verb.problemize", "title": "Record regressions and flaky checks as findings", "claimant": "agent", "summary": "Record each confirmed regression as a finding about the product and each flaky item as a finding against its check, in the findings to fixes shape, and propose accept-baseline for every change a person should decide.", "outcome": "Findings, and proposals for a person.", "ready_outcome": "regressions_recorded", "pinned": [ "flake-rules", "findings-to-fixes:record-checks", "findings-to-fixes:severity-guide" ], "slots": [ { "name": "comparison", "category": "noun.eval_run", "description": "Pin of the comparison." } ], "produces": [ { "name": "finding", "template": "findings-to-fixes:template.finding", "max": 20 } ], "steps": [ "Write one finding per regression with the baseline value, the three runs and a reproduction on the change.", "Write one finding of kind issue per flaky item, against the check, with its three runs.", "Propose findings-to-fixes.reproduce, propose-fix, fix, verify-fix and add-regression-guard for each finding, and accept-baseline for each change.", "Run the findings to fixes record checks on every finding before submitting." ], "acceptance": [ { "id": "findings-match-template", "check": "Every finding validates against template.finding's schema.", "method": "deterministic" }, { "id": "record-checks-pass", "check": "Every finding passes the findings to fixes record checks.", "method": "deterministic" }, { "id": "every-regression-recorded", "check": "Every regression and flaky item in the comparison belongs to exactly one finding.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 1200 }, "ready_requires": [ "proposed_verbs" ], "follows": [ "regression-watch.compare-to-baseline" ] }, { "slug": "review-flaky-checks", "kind": "verb.distill_reflex", "title": "Review which checks keep flaking", "claimant": "agent", "summary": "Across the comparisons in a window, count each check's flakes and runs, and propose validating the check, fixing its environment, or a wider tolerance for a person to accept.", "outcome": "A flake review with a proposal per check.", "ready_outcome": "flakes_reviewed", "model_use": "none", "pinned": [ "flake-rules", "checks-of-checks:method" ], "slots": [ { "name": "comparisons", "category": "noun.eval_run", "description": "Pins of the comparisons in the window.", "many": true, "max": 64 } ], "produces": [ { "name": "flake_review", "template": "template.flake-review" } ], "steps": [ "Count flakes and runs per check from the pinned comparisons only.", "Propose checks-of-checks.validate-checks for a check that flaked three or more times in thirty days.", "Propose a wider tolerance only as accept-baseline for a person, never as a change to the rules." ], "acceptance": [ { "id": "review-matches-template", "check": "The review validates against template.flake-review's schema.", "method": "deterministic" }, { "id": "counts-recompute", "check": "Each check's flakes and runs recompute from the pinned comparisons.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 1200 }, "follows": [ "regression-watch.compare-to-baseline" ] } ] }