{ "caveats": [ "Evaluates 12 representative 1080p frames and 22 synthetic Kokoro-82M narration lines from the owned Eranos showcase video.", "Greedy CTC without a language model measures acoustic character alignment (mean CER 0.016), not prosody or human naturalness.", "Runs strictly in an optional GPU worker; Tessera's server imports no vision or audio model." ], "hardware": { "cuda_version": "13.2", "device": "cuda", "gpu_name": "NVIDIA GeForce RTX 3060", "peak_allocated_bytes": 2079942656, "peak_reserved_bytes": 2453667840, "torch_version": "2.15.0+cu132", "total_vram_bytes": 12481003520 }, "purpose": "Worker-side multimodal QA combining Meta SAM (UI-vs-caption spatial overlap on 12 representative frames) and Meta wav2vec2-base-960h (greedy CTC speech-to-caption verification across 22 narration lines).", "receipt_sha256": "8671ba5a618ee7c5dfb3d7984e0929bffb4e636fddfee5042d74542411a023fb", "sam_ui_caption_qa": { "caption_safe_band_spec": { "x0": 240, "x1": 1680, "y0": 836, "y1": 1015 }, "frames_evaluated": 12, "model_id": "facebook/sam-vit-base", "model_weights_sha256": "892c410e496344e527255ccdcb2cb7244a609acb5389c7c4fdba1288f861c579", "per_frame": [ { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 660, "id": "v2-collide-01", "latency_ms": 2711.58, "mean_sam_iou": 0.9445, "prompt_points": 14, "regime": "v2_caption_colliding", "segmented_ui_components": 13, "shot": "nonprofit_need_and_sort", "timestamp_s": 22.0, "ui_frame_coverage": 0.04616 }, { "caption_band_fg_ui_overlap_ratio": 0.186918, "caption_band_fg_ui_pixels": 48180, "caption_band_mask_union_ratio": 0.197269, "caption_collision_detected": true, "frame_idx": 900, "id": "v2-collide-02", "latency_ms": 866.86, "mean_sam_iou": 0.9349, "prompt_points": 14, "regime": "v2_caption_colliding", "segmented_ui_components": 14, "shot": "volunteer_claim_terminal", "timestamp_s": 30.0, "ui_frame_coverage": 0.03072 }, { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 1800, "id": "v2-collide-03", "latency_ms": 816.55, "mean_sam_iou": 0.8894, "prompt_points": 14, "regime": "v2_caption_colliding", "segmented_ui_components": 12, "shot": "published_kit_panel", "timestamp_s": 60.0, "ui_frame_coverage": 0.01663 }, { "caption_band_fg_ui_overlap_ratio": 0.659226, "caption_band_fg_ui_pixels": 169922, "caption_band_mask_union_ratio": 0.683667, "caption_collision_detected": true, "frame_idx": 1980, "id": "v2-collide-04", "latency_ms": 947.48, "mean_sam_iou": 0.9318, "prompt_points": 14, "regime": "v2_caption_colliding", "segmented_ui_components": 14, "shot": "eranos_marketplace_cards", "timestamp_s": 66.0, "ui_frame_coverage": 0.0936 }, { "caption_band_fg_ui_overlap_ratio": 0.839777, "caption_band_fg_ui_pixels": 216461, "caption_band_mask_union_ratio": 0.85379, "caption_collision_detected": true, "frame_idx": 3540, "id": "v2-collide-05", "latency_ms": 874.02, "mean_sam_iou": 0.9596, "prompt_points": 14, "regime": "v2_caption_colliding", "segmented_ui_components": 14, "shot": "fixes_and_retrieval_slide", "timestamp_s": 118.0, "ui_frame_coverage": 0.16791 }, { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 5160, "id": "v2-collide-06", "latency_ms": 1168.25, "mean_sam_iou": 0.9231, "prompt_points": 14, "regime": "v2_caption_colliding", "segmented_ui_components": 8, "shot": "reveal_board_and_end_card", "timestamp_s": 172.0, "ui_frame_coverage": 0.06128 }, { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 540, "id": "safe-band-01", "latency_ms": 1109.88, "mean_sam_iou": 0.9311, "prompt_points": 13, "regime": "safe_band_y836", "segmented_ui_components": 12, "shot": "nonprofit_box_framed", "timestamp_s": 18.0, "ui_frame_coverage": 0.0634 }, { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 1080, "id": "safe-band-02", "latency_ms": 826.41, "mean_sam_iou": 0.883, "prompt_points": 14, "regime": "safe_band_y836", "segmented_ui_components": 1, "shot": "single_lease_receipt_view", "timestamp_s": 36.0, "ui_frame_coverage": 0.00096 }, { "caption_band_fg_ui_overlap_ratio": 0.224562, "caption_band_fg_ui_pixels": 57883, "caption_band_mask_union_ratio": 0.224562, "caption_collision_detected": true, "frame_idx": 1380, "id": "safe-band-03", "latency_ms": 684.43, "mean_sam_iou": 0.9014, "prompt_points": 14, "regime": "safe_band_y836", "segmented_ui_components": 13, "shot": "non_self_review_gate", "timestamp_s": 46.0, "ui_frame_coverage": 0.29583 }, { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 1620, "id": "safe-band-04", "latency_ms": 737.66, "mean_sam_iou": 0.9498, "prompt_points": 14, "regime": "safe_band_y836", "segmented_ui_components": 12, "shot": "steward_signoff_card", "timestamp_s": 54.0, "ui_frame_coverage": 0.26805 }, { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 2400, "id": "safe-band-05", "latency_ms": 702.32, "mean_sam_iou": 0.9831, "prompt_points": 14, "regime": "safe_band_y836", "segmented_ui_components": 14, "shot": "nouns_and_verbs_diagram", "timestamp_s": 80.0, "ui_frame_coverage": 0.15355 }, { "caption_band_fg_ui_overlap_ratio": 0.0, "caption_band_fg_ui_pixels": 0, "caption_band_mask_union_ratio": 0.0, "caption_collision_detected": false, "frame_idx": 2640, "id": "safe-band-06", "latency_ms": 539.46, "mean_sam_iou": 0.9053, "prompt_points": 10, "regime": "safe_band_y836", "segmented_ui_components": 8, "shot": "atomic_tile_flow", "timestamp_s": 88.0, "ui_frame_coverage": 0.00901 } ], "summary": { "mean_sam_predicted_iou": 0.9281, "p50_latency_ms_per_frame": 846.63, "safe_band_collisions_flagged": 1, "safe_band_frames_n": 6, "safe_band_mean_fg_ui_overlap_ratio": 0.037427, "safe_band_mean_mask_union_ratio": 0.037427, "v2_colliding_frames_n": 6, "v2_collisions_flagged": 3, "v2_mean_fg_ui_overlap_ratio": 0.280987, "v2_mean_mask_union_ratio": 0.289121 }, "video_sha256": "14aa3f163a434436dfa1447c179ac921edcfe37880f27d5d8c95aac687d24030" }, "schema": "tessera.multimodal-sam-asr-eval.v1", "wall_seconds": 46.551, "wav2vec2_ctc_qa": { "decoding": "greedy_ctc_no_lm", "lines": [ { "cer_no_spaces": 0.0, "duration_s": 4.06, "latency_ms": 120.81, "line": "l01", "n": 1, "reference": "IF YOUR AI PLAN HAS TOKENS LEFT OVER AT THE END OF THE MONTH", "transcript": "IF YOUR A I PLAN HAS TOKENS LEFT OVER AT THE END OF THE MONTH", "wav_file": "l01.wav", "wav_sha256": "13d4d8635de8b10c26601d880766e4a383b100ae5503c9aec89c954858af2d2a", "wer": 0.143 }, { "cer_no_spaces": 0.02, "duration_s": 4.81, "latency_ms": 19.33, "line": "l02g", "n": 2, "reference": "ERANOS PUTS THEM TO WORK FOR NONPROFITS SPARE TOKENS FOR GOOD", "transcript": "ERONOS PUTS THEM TO WORK FOR NON PROFITS SPARE TOKENS FOR GOOD", "wav_file": "l02g.wav", "wav_sha256": "2b16456b9bbb53867abcaa39dd580767d9d99ea5a672badfe52c8bc85f267260", "wer": 0.273 }, { "cer_no_spaces": 0.108, "duration_s": 3.33, "latency_ms": 17.31, "line": "l03a", "n": 3, "reference": "NONPROFITS SAY WHAT THEY NEED IN PLAIN WORDS", "transcript": "NONE PROPHIT SAY WHAT THEY NEED IN PLAIN WORDS", "wav_file": "l03a.wav", "wav_sha256": "9e135bbfdfb537bab168d153eba958e7edcb1fe4fbd6fb1eecfadfa2eaf9ccbb", "wer": 0.25 }, { "cer_no_spaces": 0.0, "duration_s": 7.51, "latency_ms": 27.71, "line": "l03r", "n": 4, "reference": "PUBLIC RESEARCH COMES BACK FINISHED AND CITED WORK ON THEIR OWN FILES BECOMES A KIT THEIR OWN AGENT RUNS", "transcript": "PUBLIC RESEARCH COMES BACK FINISHED AND CITED WORK ON THEIR OWN FILES BECOMES A KIT THEIR OWN AGENT RUNS", "wav_file": "l03r.wav", "wav_sha256": "53b984f03de6370de0e443346518ab584a43b64c95295353054712736546db74", "wer": 0.0 }, { "cer_no_spaces": 0.048, "duration_s": 6.54, "latency_ms": 25.36, "line": "l04", "n": 5, "reference": "ANYONE CAN HELP OPEN ERANOS PICK AN OPEN TASK AND HAND IT TO YOUR OWN AI AGENT", "transcript": "ANY ONE CAN HELP OPEN AIRONOS PICK AN OPEN TASK AND HAND IT TO YOUR OWN A I AGENT", "wav_file": "l04.wav", "wav_sha256": "7d365a7469249161767803dfa6aba971c24ed38c1c19b1f9e6b8c0228d1b2dce", "wer": 0.294 }, { "cer_no_spaces": 0.0, "duration_s": 6.88, "latency_ms": 26.39, "line": "l05", "n": 6, "reference": "YOUR AGENT CLAIMS THE TASK ONLY ONE AGENT WORKS ON IT AT A TIME AND EVERY STEP GETS A RECEIPT", "transcript": "YOUR AGENT CLAIMS THE TASK ONLY ONE AGENT WORKS ON IT AT A TIME AND EVERY STEP GETS A RECEIPT", "wav_file": "l05.wav", "wav_sha256": "9a238ac2802fdfdaae85eb0161d9276b630d652ac7c61a2a111879d8849a5a1e", "wer": 0.0 }, { "cer_no_spaces": 0.0, "duration_s": 7.55, "latency_ms": 26.81, "line": "l06", "n": 7, "reference": "IT DOESN'T START FROM SCRATCH IT REUSES CHECKED PIECES FROM SEED PACKS AND TESTS THE KIT ON MADE UP CASES", "transcript": "IT DOESN'T START FROM SCRATCH IT REUSES CHECKED PIECES FROM SEED PACKS AND TESTS THE KIT ON MADE UP CASES", "wav_file": "l06.wav", "wav_sha256": "0ad9295267663029a2cf46f1ff369a41a57ced4ca02396920c81e17d97f2eed9", "wer": 0.0 }, { "cer_no_spaces": 0.014, "duration_s": 5.88, "latency_ms": 21.45, "line": "l07s", "n": 8, "reference": "A DIFFERENT VOLUNTEER'S AGENT REVIEWS IT THE BUILDER CAN NEVER APPROVE ITS OWN KIT", "transcript": "A DIFFERENT VOLUNTEERS AGENT REVIEWS IT THE BUILDER CAN NEVER APPROVE ITS OWN KIT", "wav_file": "l07s.wav", "wav_sha256": "647a159339da3459dd2279c88baac833f7a8484912b990f0a323810d79d59430", "wer": 0.071 }, { "cer_no_spaces": 0.0, "duration_s": 4.08, "latency_ms": 18.87, "line": "l08", "n": 9, "reference": "SENSITIVE KITS WAIT FOR A PERSON AT A NONPROFIT TO SIGN OFF", "transcript": "SENSITIVE KITS WAIT FOR A PERSON AT A NON PROFIT TO SIGN OFF", "wav_file": "l08.wav", "wav_sha256": "79f8d0fb06c7500b6287db336163bd5c641501952e9cc527d003fb0eea15a4c6", "wer": 0.167 }, { "cer_no_spaces": 0.011, "duration_s": 8.38, "latency_ms": 33.16, "line": "l09", "n": 10, "reference": "NOW THE NONPROFIT HAS A FREE REVIEWED KIT IT DOES THE BUSYWORK AND A PERSON MAKES EVERY DECISION ABOUT A PERSON", "transcript": "NOW THE NON PROFHIT HAS A FREE REVIEWED KIT IT DOES THE BUSY WORK AND A PERSON MAKES EVERY DECISION ABOUT A PERSON", "wav_file": "l09.wav", "wav_sha256": "6651e64ca2667b712cd6f7525dea9c6a653f5bd31c8e39278b3736676d340261", "wer": 0.19 }, { "cer_no_spaces": 0.02, "duration_s": 9.01, "latency_ms": 33.19, "line": "l11b", "n": 11, "reference": "THIS ENTIRE VIDEO WAS MADE FROM NOUNS AND VERBS CLAIMED AND FINISHED BY WORKERS ON TESSERA WITH A RECEIPT FOR EVERY STEP", "transcript": "THIS ENTIRE VIDIO WAS MADE FROM NOUNS AND VERBS CLAIMED AND FINISHED BY WORKERS ON A TESSERA WITH A RECEIPT FOR EVERY STEP", "wav_file": "l11b.wav", "wav_sha256": "649f10994b509408e0c121c6d23117e570fe446ef250d5f9daa3776eb94983f0", "wer": 0.091 }, { "cer_no_spaces": 0.027, "duration_s": 3.04, "latency_ms": 19.03, "line": "l14", "n": 12, "reference": "START AT AGORA AND POINT YOUR AGENT AT ERANOS", "transcript": "START AT AGARA AND POINT YOUR AGENT AT ERANOS", "wav_file": "l14.wav", "wav_sha256": "6ba6b8956cdec4fe7e5c69a80061f7f49ab5a99fed38b36d921b1e792f25451d", "wer": 0.111 }, { "cer_no_spaces": 0.0, "duration_s": 3.98, "latency_ms": 21.56, "line": "l20", "n": 13, "reference": "AI IS SMART THE WAY WE USE IT IS FRUSTRATINGLY DUMB", "transcript": "A I IS SMART THE WAY WE USE IT IS FRUSTRATINGLY DUMB", "wav_file": "l20.wav", "wav_sha256": "f242f3e58244decc7976fbf88880d5d91385c616ccc0da31a783d8eb792b0f7c", "wer": 0.182 }, { "cer_no_spaces": 0.0, "duration_s": 8.91, "latency_ms": 28.9, "line": "l21", "n": 14, "reference": "IT DOESN'T KNOW WHAT'S NEW IT REWRITES WHAT ALREADY EXISTS IT NEEDS BABYSITTING ON LONG JOBS AND TRAINING ALONE DOESN'T FIX IT", "transcript": "IT DOESN'T KNOW WHAT'S NEW IT REWRITES WHAT ALREADY EXISTS IT NEEDS BABY SITTING ON LONG JOBS AND TRAINING ALONE DOESN'T FIX IT", "wav_file": "l21.wav", "wav_sha256": "d0edff055ea2ec980e34035002f331e57932a384dcf5ad752c8583b130168e69", "wer": 0.091 }, { "cer_no_spaces": 0.016, "duration_s": 13.18, "latency_ms": 44.06, "line": "l22", "n": 15, "reference": "EVERYTHING IS NOUNS AND VERBS NOUNS ARE REUSABLE THINGS CODE KNOWLEDGE RESULTS EVEN AN AGENT'S WHOLE SETUP VERBS ARE JOBS ANY AGENT CAN CLAIM DO AND PROVE", "transcript": "EVERYTHING IS NOUNS AND VERBS NOUNS ARE REUSIBLE THINGS CODE KNOWLEDGE RESULTS EVEN IN AGENT'S WHOLE SET UP VERBS ARE JOBS ANY AGENT CAN CLAIM DO AND PROVE", "wav_file": "l22.wav", "wav_sha256": "22053c72b464ee628e003dfa378bc34a74dcb49528a8dca28c010d5d3adfa15f", "wer": 0.148 }, { "cer_no_spaces": 0.007, "duration_s": 14.61, "latency_ms": 49.71, "line": "l23", "n": 16, "reference": "FEEDS TURN WHAT'S NEW INTO NOUNS SEARCH AND RETRIEVAL BRING THE RIGHT ONES TO ANY AGENT AS IT WORKS AND HERMA THE SHIM CAN ANSWER WITH NO MODEL CALL SO AGENTS REUSE INSTEAD OF REWRITING", "transcript": "FEEDS TURN WHAT'S NEW INTO NOWNS SEARCH AND RETRIEVAL BRING THE RIGHT ONES TO ANY AGENT AS IT WORKS AND HERMA THE SHIM CAN ANSWER WITH NO MODEL CALL SO AGENTS REUSE INSTEAD OF REWRITING", "wav_file": "l23.wav", "wav_sha256": "7798cf6d0c605993b18490fb13a4a36a2f999e8feb346763372465dc8612db29", "wer": 0.029 }, { "cer_no_spaces": 0.011, "duration_s": 7.68, "latency_ms": 26.26, "line": "l23b", "n": 17, "reference": "AND TESSERA SPLITS BIG JOBS INTO SMALL VERBS THAT MANY WORKERS CLAIM CHECK AND FINISH EACH WITH A RECEIPT", "transcript": "AND A TESSERA SPLITS BIG JOBS INTO SMALL VERBS THAT MANY WORKERS CLAIM CHECK AND FINISH EACH WITH A RECEIPT", "wav_file": "l23b.wav", "wav_sha256": "094212fac5e6d1978ca6e21512cb069a883aa5c920527f303717ff32f69f2c90", "wer": 0.053 }, { "cer_no_spaces": 0.006, "duration_s": 16.56, "latency_ms": 54.9, "line": "l24", "n": 18, "reference": "TRAINING STILL HELPS ON ONE SYNTHETIC TASK EACH SMALL LLAMA MODELS WENT FROM TWELVE TO FIFTY ONE PERCENT AND FIFTY TWO TO SEVENTY THREE AND GLIMMER THIRTY B FROM NINETY THREE TO NINETY NINE ON A FRESH PRE REGISTERED TEST", "transcript": "TRAINING STILL HELPS ON ONE SYNTHETIC TASK EACH SMALL LAMA MODELS WENT FROM TWELVE TO FIFTY ONE PER CENT AND FIFTY TWO TO SEVENTY THREE AND GLIMMER THIRTY B FROM NINETY THREE TO NINETY NINE ON A FRESH PREREGISTERED TEST", "wav_file": "l24.wav", "wav_sha256": "f239d8e9ed4ae7c5b9ddb11dd792ab9f21835021d5df8921e11f7ee867f2108a", "wer": 0.125 }, { "cer_no_spaces": 0.057, "duration_s": 8.61, "latency_ms": 28.86, "line": "l25", "n": 19, "reference": "TWO SURFACES ERANOS AND AGORA TWO PIECES OF INFRASTRUCTURE TESSERA AND HERMA AND SEED PACKS TO SHARE THEM", "transcript": "TWO SURFACES ERONOS AND AGARA TWO PIECES OF INFR STRUCTURE ETESERA AND HERMA AND SEED PACKS TO SHARE THEM", "wav_file": "l25.wav", "wav_sha256": "e900f7a78900923024375e8f29fe0a576da87ef4b9170d1d59125180d944076b", "wer": 0.278 }, { "cer_no_spaces": 0.0, "duration_s": 7.88, "latency_ms": 26.31, "line": "l30", "n": 20, "reference": "THE SAME NOUNS AND VERBS FIT ALMOST ANY PROJECT GRANT REPORTS LITERATURE REVIEWS CHECKED DATA PIPELINES", "transcript": "THE SAME NOUNS AND VERBS FIT ALMOST ANY PROJECT GRANT REPORTS LITERATURE REVIEWS CHECKED DATA PIPE LINES", "wav_file": "l30.wav", "wav_sha256": "8164ad3ff3f4d43c449cf31c770e8a0a44b9a2553c0c40c499c77f98d15fcc2e", "wer": 0.125 }, { "cer_no_spaces": 0.01, "duration_s": 9.37, "latency_ms": 30.51, "line": "l31", "n": 21, "reference": "TO CONNECT ANY AGENT COPY THE INSTRUCTIONS FROM AGORA IT FINDS THE VERBS IT CAN DO CLAIMS ONE AND REPORTS BACK WITH A RECEIPT", "transcript": "TO CONNECT ANY AGENT COPY THE INSTRUCTIONS FROM AGARA IT FINDS THE VERBS IT CAN DO CLAIMS ONE AND REPORTS BACK WITH A RECEIPT", "wav_file": "l31.wav", "wav_sha256": "7f59e67a73d3f469532d6f20636cf22aeeb55c6c7cb6be4b3883f697f00d1541", "wer": 0.042 }, { "cer_no_spaces": 0.0, "duration_s": 2.5, "latency_ms": 13.66, "line": "l32", "n": 22, "reference": "OR MAKE YOUR OWN SEED PACK AND SUBMIT IT", "transcript": "OR MAKE YOUR OWN SEED PACK AND SUBMIT IT", "wav_file": "l32.wav", "wav_sha256": "1b3f38f83ae96614495c73d19f796946aaaaaf9d2acd71872fe5301661dcb809", "wer": 0.0 } ], "lines_evaluated": 22, "mean_cer_no_spaces": 0.016, "mean_wer": 0.121, "model_id": "facebook/wav2vec2-base-960h", "model_weights_sha256": "8aa76ab2243c81747a1f832954586bc566090c83a0ac167df6f31f0fa917d74a", "p50_latency_ms_per_line": 26.6, "reference_sha256": "1fa791c0db35908e56c7d528577cc7e50970967cd7b8d4519db5fbf7b2f19c04", "zero_cer_lines": 9 } }