{ "pack_id": "dora-spinquant-and-adapter-merging", "title": "DoRA, SpinQuant and adapter merging", "version": "0.1.0", "summary": "A collection of nouns and verbs for Walsh-Hadamard outlier rotation quantization, weight-decomposed low-rank adapter analysis and subspace interference routing across static fusion and co-resident switching.", "dependencies": [ "configuration-grids-and-bounds", "evaluation-design", "wiring-and-conformance-checks" ], "boundaries": [ "FastWalshHadamardOutlierRotator applies orthogonal Sylvester Hadamard rotations over power-of-two channel dimensions and requires identical rotation seeds on inverse reconstruction.", "SubspaceInterferenceAndMergeRouter recommends ties_static_merge only when both Frobenius cosine similarity and TIES sign-conflict density stay within configuration grid bounds; otherwise adapters remain separate under dynamic_co_resident_switch.", "All quantization bit-widths, TIES trim densities, interference thresholds and numerical tolerances resolve from default_grid_spec.json and pass ZeroHardcodedValuesAstGate.", "Importing the pack writes nothing, enqueues nothing and installs no code." ], "nouns": [ { "slug": "pack", "category": "noun.skill_pack", "role": "pack", "title": "A collection of nouns and verbs for Walsh-Hadamard outlier rotation quantization, weight-decomposed low-rank adapter analysis and subspace interference routing across static fusion and co-resident switching", "value": { "lets_workers": "Suppress heavy-tailed weight and activation channel outliers via orthogonal Walsh-Hadamard rotations with deterministic Rademacher sign flips before 4-bit symmetric quantization, decompose low-rank (B @ A) updates into directional unit vectors and column magnitudes (DoRA), and route multi-adapter pools between static TIES fusion and dynamic co-resident switching using Frobenius cosine and sign-conflict density thresholds.", "flow": [ { "verb": { "$verb": "dora-spinquant-and-adapter-merging.rotate-outliers-and-quantize-weights" }, "when": "Apply Sylvester Walsh-Hadamard orthogonal rotation with deterministic sign flips and measure low-bit quantization MSE reduction against unrotated channel matrices." }, { "verb": { "$verb": "dora-spinquant-and-adapter-merging.decompose-dora-and-route-adapter-subspaces" }, "when": "Decompose low-rank adapter updates into column magnitude and unit-direction shifts and route adapter pairs between ties_static_merge and dynamic_co_resident_switch." }, { "verb": { "$verb": "dora-spinquant-and-adapter-merging.verify-quantization-merge-and-ast-gate" }, "when": "Independently verify orthogonal round-trip fidelity, 4-bit MSE reduction, TIES routing receipts and ZeroHardcodedValuesAstGate conformance." } ], "guardrails": [ { "$pin": "guardrail.co-resident-switch-on-high-subspace-interference" }, { "$pin": "guardrail.zero-inline-quantization-magic-literals" } ], "tool": { "$pin": "tool.adapter-quant-and-merge" }, "worked_example": "Synthetic 64-channel heavy-tailed outlier matrix, 4x4 DoRA probe and orthogonal/conflicting adapter pairs recomputed in test_adapter_quant_and_merge.py: SpinQuant Walsh-Hadamard rotation lowers max-to-RMS from 12.578755 to 2.166179, collapses excess kurtosis from 103.842906 to -1.034782, reduces 4-bit symmetric quantization MSE by 10.039322x (0.231252 -> 0.023035), decomposes DoRA directional/magnitude deltas (ratio 0.857493), and routes orthogonal adapters to ties_static_merge while routing sign-conflicting adapters to dynamic_co_resident_switch." }, "relations": [ [ "requires_noun", "guardrail.co-resident-switch-on-high-subspace-interference" ], [ "requires_noun", "guardrail.zero-inline-quantization-magic-literals" ], [ "requires_noun", "spec.spinquant-dora-ties-grid" ], [ "requires_noun", "rules.outlier-rotation-and-interference-routing" ], [ "requires_noun", "tool.adapter-quant-and-merge" ], [ "requires_noun", "template.adapter-quant-merge-receipt" ] ] }, { "slug": "guardrail.co-resident-switch-on-high-subspace-interference", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: never statically fuse adapters whose Frobenius cosine or TIES sign-conflict density exceeds grid thresholds", "category_reason": "Prevents destructive parameter cancellation when combining conflicting task adapters.", "value": { "rule": "Route adapter pairs to ties_static_merge only when max_abs_frobenius_cosine <= max_cosine_for_static_merge and sign_conflict_density <= max_sign_conflict_for_static_merge; route interfering or sign-conflicting pairs to dynamic_co_resident_switch.", "how_it_is_checked": "SubspaceInterferenceAndMergeRouter.route_adapters enforces clamped grid bounds and unit tests verify dynamic_co_resident_switch on sign-conflicting adapters." } }, { "slug": "guardrail.zero-inline-quantization-magic-literals", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: rotation radices, quantization bit-widths, TIES densities and interference bounds forbid inline scalars", "category_reason": "Keeps quantization and adapter-routing thresholds governed by externalized configuration grids.", "value": { "rule": "Operational Python functions and classes in adapter_quant_and_merge may only inline canonical mathematical identities (-1, 0, 1, -1.0, 0.0, 1.0); all bit-widths, trim densities and thresholds resolve from default_grid_spec.json.", "how_it_is_checked": "scan_directory_for_hardcoded_values and ZeroHardcodedValuesAstGate.scan_paths scan every module in adapter_quant_and_merge during make check." } }, { "slug": "spec.spinquant-dora-ties-grid", "category": "noun.benchmark_suite", "role": "spec", "title": "Specification: FastWalshHadamardOutlierRotator, WeightDecomposedLoraAnalyzer and SubspaceInterferenceAndMergeRouter invariants", "category_reason": "Defines orthogonality, 4-bit MSE suppression, column-norm DoRA decomposition and TIES routing invariants.", "value": { "components": [ "FastWalshHadamardOutlierRotator", "WeightDecomposedLoraAnalyzer", "SubspaceInterferenceAndMergeRouter" ], "default_quant_bits": 4, "synthetic_channel_dim": 64, "ties_default_trim_density": 0.5, "max_cosine_for_static_merge": 0.3, "max_sign_conflict_for_static_merge": 0.25 } }, { "slug": "rules.outlier-rotation-and-interference-routing", "category": "noun.reflex_procedure", "role": "rules", "title": "Deterministic Walsh-Hadamard outlier rotation, DoRA decomposition and TIES interference routing rules", "category_reason": "Zero-model mathematical procedure for low-bit quantization preparation and multi-adapter serving selection.", "value": { "rotation_formula": "W_rot = (W * diag(s)) @ H_d / sqrt(d) with s_j in {-1, +1}", "dora_formula": "W_dora[:, j] = m_j * ((W_0 + scale * B @ A)[:, j] / ||(W_0 + scale * B @ A)[:, j]||_2)", "routing_decision": { "low_cosine_and_low_sign_conflict": "ties_static_merge", "high_cosine_or_sign_conflict": "dynamic_co_resident_switch" }, "model_calls": 0 } }, { "slug": "tool.adapter-quant-and-merge", "category": "noun.code_node", "role": "tool_card", "title": "Tool card: standard-library SpinQuant Walsh-Hadamard rotator, DoRA analyzer and TIES interference router", "category_reason": "Tested Python 3.10+ standard-library package pinned by path, file count and SHA-256.", "value": { "name": "DoRA, SpinQuant and adapter merging", "code": { "$code": "dora-spinquant-and-adapter-merging" }, "runtime": "Python 3.10 or later, standard library only.", "commands": [ "python -m adapter_quant_and_merge --bundle BUNDLE.json" ], "does": [ "Implements FastWalshHadamardOutlierRotator with Sylvester Hadamard butterflies, deterministic sign flips, kurtosis/max-to-RMS diagnostics and 4-bit symmetric uniform quantization MSE evaluation.", "Implements WeightDecomposedLoraAnalyzer computing column-wise magnitude and directional cosine learning ratios from (B @ A) low-rank updates.", "Implements SubspaceInterferenceAndMergeRouter computing flattened Frobenius cosine similarity, TIES trim-elect-merge sign-conflict density and static-vs-dynamic routing." ], "does_not": [ "Perform network I/O, invoke external tensor runtimes or inline hardcoded numeric thresholds." ], "code_license": "MIT" }, "relations": [ [ "verifies", "spec.spinquant-dora-ties-grid" ] ] }, { "slug": "template.adapter-quant-merge-receipt", "category": "noun.scene_recipe", "role": "template", "title": "Template: Walsh-Hadamard 4-bit quantization, DoRA decomposition and TIES interference routing receipt", "value": { "describes": "a deterministic receipt recording before/after Walsh-Hadamard outlier metrics, 4-bit quantization MSE reduction, DoRA directional/magnitude shifts and adapter interference routing", "schema": { "type": "object", "additionalProperties": false, "required": [ "unrotated_max_to_rms", "rotated_max_to_rms", "unrotated_excess_kurtosis", "rotated_excess_kurtosis", "unrotated_quant_mse", "rotated_quant_mse", "mse_reduction_factor", "dora_directional_cosine_delta", "dora_relative_magnitude_delta", "dora_directional_to_magnitude_ratio", "orthogonal_frobenius_cosine", "orthogonal_sign_conflict_density", "orthogonal_route", "conflicting_frobenius_cosine", "conflicting_sign_conflict_density", "conflicting_route" ], "properties": { "unrotated_max_to_rms": { "type": "number", "minimum": 0.0 }, "rotated_max_to_rms": { "type": "number", "minimum": 0.0 }, "unrotated_excess_kurtosis": { "type": "number" }, "rotated_excess_kurtosis": { "type": "number" }, "unrotated_quant_mse": { "type": "number", "minimum": 0.0 }, "rotated_quant_mse": { "type": "number", "minimum": 0.0 }, "mse_reduction_factor": { "type": "number", "minimum": 1.0 }, "dora_directional_cosine_delta": { "type": "number", "minimum": 0.0 }, "dora_relative_magnitude_delta": { "type": "number", "minimum": 0.0 }, "dora_directional_to_magnitude_ratio": { "type": "number", "minimum": 0.0 }, "orthogonal_frobenius_cosine": { "type": "number", "minimum": 0.0, "maximum": 1.0 }, "orthogonal_sign_conflict_density": { "type": "number", "minimum": 0.0, "maximum": 1.0 }, "orthogonal_route": { "type": "string", "enum": [ "ties_static_merge", "dynamic_co_resident_switch" ] }, "conflicting_frobenius_cosine": { "type": "number", "minimum": 0.0, "maximum": 1.0 }, "conflicting_sign_conflict_density": { "type": "number", "minimum": 0.0, "maximum": 1.0 }, "conflicting_route": { "type": "string", "enum": [ "ties_static_merge", "dynamic_co_resident_switch" ] } } }, "example": { "unrotated_max_to_rms": 12.578755, "rotated_max_to_rms": 2.166179, "unrotated_excess_kurtosis": 103.842906, "rotated_excess_kurtosis": -1.034782, "unrotated_quant_mse": 0.231252, "rotated_quant_mse": 0.023035, "mse_reduction_factor": 10.039322, "dora_directional_cosine_delta": 0.142507, "dora_relative_magnitude_delta": 0.16619, "dora_directional_to_magnitude_ratio": 0.857493, "orthogonal_frobenius_cosine": 0.0, "orthogonal_sign_conflict_density": 0.0, "orthogonal_route": "ties_static_merge", "conflicting_frobenius_cosine": 0.999793, "conflicting_sign_conflict_density": 1.0, "conflicting_route": "dynamic_co_resident_switch" } } }, { "slug": "example.adapter-quant-merge-bundle", "category": "noun.eval_run", "role": "example", "synthetic": true, "title": "Worked example input: 64-channel heavy-tailed outlier configuration, 4x4 DoRA probe and orthogonal/conflicting adapters", "category_reason": "Synthetic fixture recomputed by test_adapter_quant_and_merge.py.", "value": { "rotation_seed": 1729, "quant_bits": 4, "synthetic_rows": 16, "synthetic_cols": 64, "synthetic_outlier_channels": 1, "synthetic_outlier_scale": 32.0, "synthetic_seed": 42, "dora_probe": { "base_weights": [ [ 1.0, 0.0, 1.0, 0.0 ], [ 0.0, 1.0, 0.0, 1.0 ], [ 0.0, 0.0, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ] ], "lora_b": [ [ 0.0, 0.0 ], [ 0.0, 0.0 ], [ 0.6, 0.0 ], [ 0.0, 0.6 ] ], "lora_a": [ [ 1.0, 0.0, -1.0, 0.0 ], [ 0.0, 1.0, 0.0, -1.0 ] ], "lora_scale": 1.0 }, "orthogonal_adapters": [ [ [ 0.8, 0.0, 0.0, 0.0 ], [ 0.6, 0.0, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ] ], [ [ 0.0, 0.0, 0.7, 0.0 ], [ 0.0, 0.0, 0.0, -0.9 ], [ 0.0, 0.0, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ] ] ], "conflicting_adapters": [ [ [ 0.9, -0.8, 0.7, -0.6 ], [ 0.5, -0.4, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ] ], [ [ -0.85, 0.75, -0.65, 0.55 ], [ -0.45, 0.35, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0 ] ] ], "shows": "Synthetic heavy-tailed channel quantization, DoRA decomposition and multi-adapter interference routing workload.", "source": "Synthetic. Written for this pack." } }, { "slug": "example.adapter-quant-merge-receipt", "category": "noun.eval_run", "role": "example", "synthetic": true, "title": "Worked example output: recomputed 10.04x SpinQuant 4-bit MSE reduction, DoRA ratio and TIES routing receipt", "category_reason": "Synthetic receipt verified by test_adapter_quant_and_merge.py.", "value": { "conforms_to": { "$pin": "template.adapter-quant-merge-receipt" }, "instance": { "unrotated_max_to_rms": 12.578755, "rotated_max_to_rms": 2.166179, "unrotated_excess_kurtosis": 103.842906, "rotated_excess_kurtosis": -1.034782, "unrotated_quant_mse": 0.231252, "rotated_quant_mse": 0.023035, "mse_reduction_factor": 10.039322, "dora_directional_cosine_delta": 0.142507, "dora_relative_magnitude_delta": 0.16619, "dora_directional_to_magnitude_ratio": 0.857493, "orthogonal_frobenius_cosine": 0.0, "orthogonal_sign_conflict_density": 0.0, "orthogonal_route": "ties_static_merge", "conflicting_frobenius_cosine": 0.999793, "conflicting_sign_conflict_density": 1.0, "conflicting_route": "dynamic_co_resident_switch" }, "shows": "Recomputes 10.039322x 4-bit quantization MSE reduction via Walsh-Hadamard outlier rotation, DoRA directional/magnitude update decomposition and interference-aware adapter routing.", "source": "Synthetic. Recomputed directly from example.adapter-quant-merge-bundle." }, "relations": [ [ "derives_from", "example.adapter-quant-merge-bundle" ], [ "verifies", "template.adapter-quant-merge-receipt" ] ] } ], "verbs": [ { "slug": "rotate-outliers-and-quantize-weights", "kind": "verb.generate_seed", "title": "Rotate heavy-tailed channel outliers with Walsh-Hadamard transforms and evaluate low-bit quantization", "claimant": "agent", "model_use": "none", "summary": "Apply Sylvester Walsh-Hadamard orthogonal rotations with deterministic Rademacher sign flips to disperse concentrated channel outliers before symmetric 4-bit uniform quantization.", "outcome": "Outlier kurtosis, max-to-RMS suppression and low-bit reconstruction MSE metrics.", "ready_outcome": "weights_rotated_and_quantized", "pinned": [ "spec.spinquant-dora-ties-grid", "rules.outlier-rotation-and-interference-routing", "guardrail.zero-inline-quantization-magic-literals", "tool.adapter-quant-and-merge" ], "slots": [ { "name": "weight_bundle", "category": "noun.eval_run", "description": "Pin of channel weight matrices and quantization grid parameters." } ], "produces": [ { "name": "quantization_receipt", "template": "template.adapter-quant-merge-receipt" } ], "steps": [ "Load quantization bit-width, granularity and rotation seed from default_grid_spec.json without inline magic numbers.", "Compute Fast Walsh-Hadamard rotated weights, run symmetric uniform quantization and compare inverse-rotated reconstruction MSE against unrotated quantization." ], "acceptance": [ { "id": "orthogonality-and-mse-reduction", "check": "Inverse Walsh-Hadamard rotation recovers unquantized weights within numerical precision and reduces 4-bit MSE on heavy-tailed channel matrices.", "method": "deterministic" } ] }, { "slug": "decompose-dora-and-route-adapter-subspaces", "kind": "verb.probe_source", "title": "Decompose DoRA magnitude/direction updates and route multi-adapter interference", "claimant": "agent", "model_use": "none", "summary": "Decompose low-rank (B @ A) deltas into column magnitudes and unit-direction cosine shifts, and score adapter pairs by Frobenius cosine similarity and TIES sign-conflict density to select static fusion vs co-resident switching.", "outcome": "DoRA directional-to-magnitude learning metrics and interference-aware adapter serving recommendations.", "ready_outcome": "adapters_decomposed_and_routed", "pinned": [ "spec.spinquant-dora-ties-grid", "rules.outlier-rotation-and-interference-routing", "guardrail.co-resident-switch-on-high-subspace-interference", "tool.adapter-quant-and-merge" ], "slots": [ { "name": "adapter_bundle", "category": "noun.eval_run", "description": "Pin of base weights, low-rank adapter factors and candidate multi-adapter deltas." } ], "produces": [ { "name": "adapter_routing_receipt", "template": "template.adapter-quant-merge-receipt" } ], "steps": [ "Decompose base and low-rank-updated weight columns into Euclidean magnitudes and unit directions via WeightDecomposedLoraAnalyzer.", "Evaluate flattened Frobenius cosine similarity and TIES sign-conflict density across adapter pairs to select ties_static_merge or dynamic_co_resident_switch." ], "acceptance": [ { "id": "interference-routing-safety", "check": "Orthogonal non-conflicting adapters route to ties_static_merge while overlapping sign-conflicting adapters route to dynamic_co_resident_switch.", "method": "deterministic" } ], "follows": [ "dora-spinquant-and-adapter-merging.rotate-outliers-and-quantize-weights" ] }, { "slug": "verify-quantization-merge-and-ast-gate", "kind": "verb.verify_gate", "title": "Independently verify SpinQuant MSE reduction, DoRA/TIES receipts and AST magic-literal gates", "claimant": "agent", "model_use": "none", "summary": "Recompute Walsh-Hadamard quantization MSE reduction, DoRA magnitude-direction decomposition and TIES routing verdicts under a distinct reviewer rule and verify ZeroHardcodedValuesAstGate passes every module.", "outcome": "Independent deterministic verification receipt for outlier rotation, DoRA analysis and adapter routing.", "ready_outcome": "quant_and_merge_verified", "review": "distinct_reviewer", "pinned": [ "spec.spinquant-dora-ties-grid", "guardrail.co-resident-switch-on-high-subspace-interference", "guardrail.zero-inline-quantization-magic-literals", "tool.adapter-quant-and-merge", "template.adapter-quant-merge-receipt" ], "slots": [ { "name": "eval_bundle", "category": "noun.eval_run", "description": "Pin of outlier matrix, DoRA factors and multi-adapter routing fixture." } ], "produces": [ { "name": "verified_quant_merge_receipt", "template": "template.adapter-quant-merge-receipt" } ], "steps": [ "Recompute run_bundle outputs and confirm schema conformance against the pinned receipt template.", "Run scan_directory_for_hardcoded_values across adapter_quant_and_merge and confirm zero AST magic-literal violations." ], "acceptance": [ { "id": "ast-and-receipt-recomputation", "check": "scan_directory_for_hardcoded_values returns an empty list and run_bundle reproduces the exact template-conforming receipt.", "method": "deterministic" } ], "follows": [ "dora-spinquant-and-adapter-merging.decompose-dora-and-route-adapter-subspaces" ] } ] }