| { |
| "release": "Dot-v0.4-Thinking", |
| "checkpoint_step": 938, |
| "training_tokens": 8642015, |
| "evaluation_scope": { |
| "task_families": 8, |
| "relationship_to_training": "same executable generators, separately seeded and hash-disjoint records", |
| "warning": "This is targeted curriculum evaluation, not a broad reasoning or ARC-AGI benchmark." |
| }, |
| "teacher_forced_4096": { |
| "baseline_zero_gate": { |
| "response_nll": 2.1283936659, |
| "response_perplexity": 8.40136, |
| "teacher_forced_token_accuracy": 0.666026, |
| "teacher_forced_exact_response_rate": 0.0, |
| "reasoning_nll": 2.511300, |
| "reasoning_token_accuracy": 0.604705, |
| "answer_nll": 0.0667919, |
| "answer_token_accuracy": 0.9960501 |
| }, |
| "dot_v0_4": { |
| "response_nll": 0.01736697, |
| "response_perplexity": 1.017519, |
| "teacher_forced_token_accuracy": 0.9942831, |
| "teacher_forced_exact_response_rate": 0.8776855, |
| "reasoning_nll": 0.0205677, |
| "reasoning_token_accuracy": 0.9932305, |
| "answer_nll": 0.00011697, |
| "answer_token_accuracy": 0.9999480 |
| }, |
| "response_nll_change": -2.1110267 |
| }, |
| "free_running_256": { |
| "thinking_envelope_valid_rate": 1.0, |
| "nonempty_reasoning_rate": 1.0, |
| "mean_reasoning_tokens": 23.8633, |
| "mean_total_generated_tokens": 28.5977, |
| "truncated_before_think_close_rate": 0.0, |
| "final_answer_exact_match": 0.96875, |
| "correct": 248, |
| "examples": 256 |
| }, |
| "adversarial_spatial_probe_64": { |
| "suite_relationship": "new spatial generators outside the v0.4 training curriculum", |
| "final_answer_exact_match": 0.21875, |
| "correct": 14, |
| "examples": 64, |
| "thinking_envelope_valid_rate": 0.875, |
| "truncation_rate": 0.109375, |
| "orientation_exact_match": 0.0, |
| "warning": "The larger 256-case run was stopped after a 15-minute timeout caused by uncached verbose generation." |
| } |
| } |
|
|
|
|
