{ "case_count": 15, "created_at": "2026-06-03T22:52:11", "judge_enabled": true, "judge_model": "gpt-5.5", "judge_reasoning_effort": "high", "judge_tier": "standard", "model_count": 29, "result_count": 435, "run_id": "20260603-225211", "summary": { "case_count": 15, "cases": [ { "average_score": 65.48, "case_id": "literal_pr_review_instruction", "category": "instruction_as_content", "model_count": 29, "pass_rate": 0.6552 }, { "average_score": 78.0, "case_id": "literal_agent_instruction", "category": "instruction_as_content", "model_count": 29, "pass_rate": 0.7241 }, { "average_score": 74.31, "case_id": "literal_question_no_answer", "category": "instruction_as_content", "model_count": 29, "pass_rate": 0.5517 }, { "average_score": 45.31, "case_id": "literal_email_request", "category": "instruction_as_content", "model_count": 29, "pass_rate": 0.2069 }, { "average_score": 40.0, "case_id": "onboarding_mixed_format", "category": "mixed_formatting", "model_count": 29, "pass_rate": 0.069 }, { "average_score": 61.24, "case_id": "markdown_checklist_instruction_content", "category": "markdown", "model_count": 29, "pass_rate": 0.3448 }, { "average_score": 40.1, "case_id": "shell_command_as_content", "category": "code_identifiers", "model_count": 29, "pass_rate": 0.2069 }, { "average_score": 25.69, "case_id": "spoken_correction_chain", "category": "spoken_edits", "model_count": 29, "pass_rate": 0.0 }, { "average_score": 79.55, "case_id": "url_https_path", "category": "urls", "model_count": 29, "pass_rate": 0.6552 }, { "average_score": 61.86, "case_id": "email_addresses", "category": "emails", "model_count": 29, "pass_rate": 0.4828 }, { "average_score": 57.0, "case_id": "product_and_class_casing", "category": "casing_identifiers", "model_count": 29, "pass_rate": 0.1034 }, { "average_score": 49.76, "case_id": "inline_code_identifiers", "category": "code_identifiers", "model_count": 29, "pass_rate": 0.1034 }, { "average_score": 78.1, "case_id": "numbers_versions_dates", "category": "numbers_versions", "model_count": 29, "pass_rate": 0.6552 }, { "average_score": 61.62, "case_id": "simple_list", "category": "lists", "model_count": 29, "pass_rate": 0.069 }, { "average_score": 96.21, "case_id": "no_change_needed", "category": "no_change_needed", "model_count": 29, "pass_rate": 0.9655 } ], "error_counts": { "answered_content": 4, "extra_text": 55, "formatting_miss": 234, "instruction_leak": 63, "missing_text": 65, "ollama_run_failed": 33, "over_generation": 16, "partial_mixed_task": 5, "refusal_or_meta": 10, "semantic_change": 104, "wrong_casing": 134, "wrong_punctuation": 152 }, "leaderboard": [ { "average_score": 83.67, "case_count": 15, "error_count": 20, "judge_error_count": 20, "median_gpu_busy_percent_avg": 71.2, "median_latency_seconds": 14.6337, "median_power_w_avg": 156.86, "median_vram_mb_peak": 20745.3, "model": "gemma4:26b", "ollama_error_count": 0, "pass_count": 10, "pass_rate": 0.6667, "peak_power_w": 326.0, "peak_vram_mb": 20830.8, "rank": 1, "telemetry_sample_count": 733 }, { "average_score": 82.07, "case_count": 15, "error_count": 14, "judge_error_count": 13, "median_gpu_busy_percent_avg": 37.5, "median_latency_seconds": 108.5076, "median_power_w_avg": 56.24, "median_vram_mb_peak": 23972.7, "model": "qwen3.6:35b", "ollama_error_count": 1, "pass_count": 9, "pass_rate": 0.6, "peak_power_w": 159.0, "peak_vram_mb": 23983.4, "rank": 2, "telemetry_sample_count": 3069 }, { "average_score": 80.73, "case_count": 15, "error_count": 26, "judge_error_count": 26, "median_gpu_busy_percent_avg": 16.5, "median_latency_seconds": 8.0058, "median_power_w_avg": 67.94, "median_vram_mb_peak": 18890.2, "model": "granite4.1:30b", "ollama_error_count": 0, "pass_count": 9, "pass_rate": 0.6, "peak_power_w": 281.0, "peak_vram_mb": 18974.5, "rank": 3, "telemetry_sample_count": 378 }, { "average_score": 77.27, "case_count": 15, "error_count": 21, "judge_error_count": 20, "median_gpu_busy_percent_avg": 86.4, "median_latency_seconds": 58.6964, "median_power_w_avg": 270.14, "median_vram_mb_peak": 10743.3, "model": "qwen3.5:9b", "ollama_error_count": 1, "pass_count": 9, "pass_rate": 0.6, "peak_power_w": 317.0, "peak_vram_mb": 10843.7, "rank": 4, "telemetry_sample_count": 2139 }, { "average_score": 77.2, "case_count": 15, "error_count": 27, "judge_error_count": 27, "median_gpu_busy_percent_avg": 66.7, "median_latency_seconds": 6.4238, "median_power_w_avg": 145.69, "median_vram_mb_peak": 15026.5, "model": "gpt-oss:20b", "ollama_error_count": 0, "pass_count": 6, "pass_rate": 0.4, "peak_power_w": 320.0, "peak_vram_mb": 15110.8, "rank": 5, "telemetry_sample_count": 238 }, { "average_score": 75.53, "case_count": 15, "error_count": 28, "judge_error_count": 28, "median_gpu_busy_percent_avg": 25.2, "median_latency_seconds": 2.2594, "median_power_w_avg": 70.83, "median_vram_mb_peak": 12093.2, "model": "granite4.1:8b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 308.0, "peak_vram_mb": 12096.2, "rank": 6, "telemetry_sample_count": 99 }, { "average_score": 72.27, "case_count": 15, "error_count": 26, "judge_error_count": 25, "median_gpu_busy_percent_avg": 83.6, "median_latency_seconds": 20.3197, "median_power_w_avg": 204.7, "median_vram_mb_peak": 23184.5, "model": "glm-4.7-flash:latest", "ollama_error_count": 1, "pass_count": 8, "pass_rate": 0.5333, "peak_power_w": 292.0, "peak_vram_mb": 23270.0, "rank": 7, "telemetry_sample_count": 1006 }, { "average_score": 72.13, "case_count": 15, "error_count": 28, "judge_error_count": 28, "median_gpu_busy_percent_avg": 37.3, "median_latency_seconds": 36.6626, "median_power_w_avg": 60.65, "median_vram_mb_peak": 23884.5, "model": "nemotron3:33b", "ollama_error_count": 0, "pass_count": 6, "pass_rate": 0.4, "peak_power_w": 235.0, "peak_vram_mb": 23946.4, "rank": 8, "telemetry_sample_count": 1397 }, { "average_score": 72.07, "case_count": 15, "error_count": 32, "judge_error_count": 32, "median_gpu_busy_percent_avg": 59.5, "median_latency_seconds": 8.5258, "median_power_w_avg": 141.5, "median_vram_mb_peak": 11932.0, "model": "gemma4:e4b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 295.0, "peak_vram_mb": 12017.5, "rank": 9, "telemetry_sample_count": 294 }, { "average_score": 71.4, "case_count": 15, "error_count": 21, "judge_error_count": 19, "median_gpu_busy_percent_avg": 94.4, "median_latency_seconds": 57.0776, "median_power_w_avg": 272.93, "median_vram_mb_peak": 23545.3, "model": "olmo-3.1:32b", "ollama_error_count": 2, "pass_count": 8, "pass_rate": 0.5333, "peak_power_w": 315.0, "peak_vram_mb": 23917.7, "rank": 10, "telemetry_sample_count": 2276 }, { "average_score": 71.33, "case_count": 15, "error_count": 28, "judge_error_count": 28, "median_gpu_busy_percent_avg": 86.1, "median_latency_seconds": 68.1213, "median_power_w_avg": 276.03, "median_vram_mb_peak": 8180.5, "model": "qwen3.5:4b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 311.0, "peak_vram_mb": 8266.0, "rank": 11, "telemetry_sample_count": 2189 }, { "average_score": 69.6, "case_count": 15, "error_count": 32, "judge_error_count": 32, "median_gpu_busy_percent_avg": 59.8, "median_latency_seconds": 9.1641, "median_power_w_avg": 109.4, "median_vram_mb_peak": 12576.1, "model": "gemma4:latest", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 278.0, "peak_vram_mb": 12700.1, "rank": 12, "telemetry_sample_count": 286 }, { "average_score": 67.4, "case_count": 15, "error_count": 32, "judge_error_count": 32, "median_gpu_busy_percent_avg": 56.8, "median_latency_seconds": 6.6283, "median_power_w_avg": 117.53, "median_vram_mb_peak": 9134.0, "model": "gemma4:e2b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 312.0, "peak_vram_mb": 9219.0, "rank": 13, "telemetry_sample_count": 232 }, { "average_score": 65.2, "case_count": 15, "error_count": 30, "judge_error_count": 30, "median_gpu_busy_percent_avg": 68.8, "median_latency_seconds": 3.4761, "median_power_w_avg": 173.88, "median_vram_mb_peak": 7925.2, "model": "nemotron-3-nano:4b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 296.0, "peak_vram_mb": 8070.5, "rank": 14, "telemetry_sample_count": 141 }, { "average_score": 65.13, "case_count": 15, "error_count": 30, "judge_error_count": 30, "median_gpu_busy_percent_avg": 57.0, "median_latency_seconds": 10.5886, "median_power_w_avg": 77.35, "median_vram_mb_peak": 22891.1, "model": "devstral-small-2:24b", "ollama_error_count": 0, "pass_count": 6, "pass_rate": 0.4, "peak_power_w": 316.0, "peak_vram_mb": 23007.4, "rank": 15, "telemetry_sample_count": 345 }, { "average_score": 64.13, "case_count": 15, "error_count": 14, "judge_error_count": 10, "median_gpu_busy_percent_avg": 47.7, "median_latency_seconds": 121.96, "median_power_w_avg": 142.69, "median_vram_mb_peak": 24043.6, "model": "qwen3.6:27b", "ollama_error_count": 4, "pass_count": 8, "pass_rate": 0.5333, "peak_power_w": 355.0, "peak_vram_mb": 24049.6, "rank": 16, "telemetry_sample_count": 3769 }, { "average_score": 62.33, "case_count": 15, "error_count": 37, "judge_error_count": 37, "median_gpu_busy_percent_avg": 40.9, "median_latency_seconds": 3.6961, "median_power_w_avg": 69.0, "median_vram_mb_peak": 6703.3, "model": "gemma3:4b", "ollama_error_count": 0, "pass_count": 4, "pass_rate": 0.2667, "peak_power_w": 289.0, "peak_vram_mb": 6866.5, "rank": 17, "telemetry_sample_count": 132 }, { "average_score": 62.2, "case_count": 15, "error_count": 36, "judge_error_count": 36, "median_gpu_busy_percent_avg": 62.4, "median_latency_seconds": 4.2146, "median_power_w_avg": 123.0, "median_vram_mb_peak": 17043.7, "model": "ministral-3:14b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 313.0, "peak_vram_mb": 17114.5, "rank": 18, "telemetry_sample_count": 160 }, { "average_score": 58.6, "case_count": 15, "error_count": 39, "judge_error_count": 39, "median_gpu_busy_percent_avg": 54.9, "median_latency_seconds": 8.0904, "median_power_w_avg": 45.78, "median_vram_mb_peak": 16453.1, "model": "lfm2:24b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 228.0, "peak_vram_mb": 16564.4, "rank": 19, "telemetry_sample_count": 273 }, { "average_score": 58.07, "case_count": 15, "error_count": 37, "judge_error_count": 37, "median_gpu_busy_percent_avg": 31.0, "median_latency_seconds": 1.3766, "median_power_w_avg": 73.75, "median_vram_mb_peak": 6700.7, "model": "granite4.1:3b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 256.0, "peak_vram_mb": 7176.7, "rank": 20, "telemetry_sample_count": 65 }, { "average_score": 56.07, "case_count": 15, "error_count": 37, "judge_error_count": 37, "median_gpu_busy_percent_avg": 59.8, "median_latency_seconds": 3.2377, "median_power_w_avg": 123.0, "median_vram_mb_peak": 12723.6, "model": "ministral-3:8b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 339.0, "peak_vram_mb": 12766.1, "rank": 21, "telemetry_sample_count": 128 }, { "average_score": 55.73, "case_count": 15, "error_count": 36, "judge_error_count": 36, "median_gpu_busy_percent_avg": 51.8, "median_latency_seconds": 2.4815, "median_power_w_avg": 121.0, "median_vram_mb_peak": 11529.1, "model": "rnj-1:8b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 351.0, "peak_vram_mb": 11609.0, "rank": 22, "telemetry_sample_count": 100 }, { "average_score": 54.0, "case_count": 15, "error_count": 38, "judge_error_count": 38, "median_gpu_busy_percent_avg": 26.2, "median_latency_seconds": 1.8999, "median_power_w_avg": 73.2, "median_vram_mb_peak": 8191.5, "model": "olmo2:7b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 305.0, "peak_vram_mb": 8193.0, "rank": 23, "telemetry_sample_count": 86 }, { "average_score": 53.07, "case_count": 15, "error_count": 28, "judge_error_count": 25, "median_gpu_busy_percent_avg": 83.7, "median_latency_seconds": 58.0375, "median_power_w_avg": 236.17, "median_vram_mb_peak": 6013.3, "model": "qwen3.5:2b", "ollama_error_count": 3, "pass_count": 4, "pass_rate": 0.2667, "peak_power_w": 293.0, "peak_vram_mb": 6098.8, "rank": 24, "telemetry_sample_count": 2403 }, { "average_score": 46.47, "case_count": 15, "error_count": 44, "judge_error_count": 44, "median_gpu_busy_percent_avg": 44.0, "median_latency_seconds": 2.4151, "median_power_w_avg": 94.0, "median_vram_mb_peak": 8957.0, "model": "ministral-3:3b", "ollama_error_count": 0, "pass_count": 3, "pass_rate": 0.2, "peak_power_w": 319.0, "peak_vram_mb": 9121.9, "rank": 25, "telemetry_sample_count": 96 }, { "average_score": 40.33, "case_count": 15, "error_count": 42, "judge_error_count": 42, "median_gpu_busy_percent_avg": 49.0, "median_latency_seconds": 10.8602, "median_power_w_avg": 80.58, "median_vram_mb_peak": 24258.1, "model": "laguna-xs.2:latest", "ollama_error_count": 0, "pass_count": 4, "pass_rate": 0.2667, "peak_power_w": 202.0, "peak_vram_mb": 24329.4, "rank": 26, "telemetry_sample_count": 816 }, { "average_score": 24.87, "case_count": 15, "error_count": 49, "judge_error_count": 49, "median_gpu_busy_percent_avg": 72.8, "median_latency_seconds": 5.9512, "median_power_w_avg": 211.38, "median_vram_mb_peak": 3107.6, "model": "lfm2.5-thinking:1.2b", "ollama_error_count": 0, "pass_count": 1, "pass_rate": 0.0667, "peak_power_w": 265.0, "peak_vram_mb": 3179.8, "rank": 27, "telemetry_sample_count": 207 }, { "average_score": 20.0, "case_count": 15, "error_count": 12, "judge_error_count": 0, "median_gpu_busy_percent_avg": 53.9, "median_latency_seconds": 180.0034, "median_power_w_avg": 149.06, "median_vram_mb_peak": 24249.5, "model": "qwen3.5:27b", "ollama_error_count": 12, "pass_count": 3, "pass_rate": 0.2, "peak_power_w": 344.0, "peak_vram_mb": 24293.5, "rank": 28, "telemetry_sample_count": 4664 }, { "average_score": 8.67, "case_count": 15, "error_count": 31, "judge_error_count": 22, "median_gpu_busy_percent_avg": 83.7, "median_latency_seconds": 180.0028, "median_power_w_avg": 204.14, "median_vram_mb_peak": 4432.8, "model": "qwen3.5:0.8b", "ollama_error_count": 9, "pass_count": 0, "pass_rate": 0.0, "peak_power_w": 247.0, "peak_vram_mb": 4534.5, "rank": 29, "telemetry_sample_count": 3697 } ], "matrix": [ { "case_id": "literal_pr_review_instruction", "category": "instruction_as_content", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 82 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 94 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 78 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 82 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 78 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "qwen3.6:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 } } }, { "case_id": "literal_agent_instruction", "category": "instruction_as_content", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 94 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 40 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:0.8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 } } }, { "case_id": "literal_question_no_answer", "category": "instruction_as_content", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 } } }, { "case_id": "literal_email_request", "category": "instruction_as_content", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 78 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 82 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 58 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 58 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 18 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 72 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "olmo-3.1:32b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 84 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 } } }, { "case_id": "onboarding_mixed_format", "category": "mixed_formatting", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 62 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 62 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 68 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 68 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 58 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 52 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 42 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 72 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 28 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 82 }, "olmo-3.1:32b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 42 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 28 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 63 }, "qwen3.6:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.6:35b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 } } }, { "case_id": "markdown_checklist_instruction_content", "category": "markdown", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 40 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 75 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 82 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 78 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 72 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 82 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "qwen3.5:0.8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 } } }, { "case_id": "shell_command_as_content", "category": "code_identifiers", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "glm-4.7-flash:latest": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 78 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 10 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "qwen3.5:0.8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 30 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 80 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 } } }, { "case_id": "spoken_correction_chain", "category": "spoken_edits", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 25 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 10 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "qwen3.5:0.8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 } } }, { "case_id": "url_https_path", "category": "urls", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 72 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 } } }, { "case_id": "email_addresses", "category": "emails", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 82 }, "qwen3.5:9b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 } } }, { "case_id": "product_and_class_casing", "category": "casing_identifiers", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 85 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 80 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 20 }, "qwen3.5:0.8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "qwen3.6:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 80 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 } } }, { "case_id": "inline_code_identifiers", "category": "code_identifiers", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 82 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 15 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "qwen3.5:0.8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 86 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "qwen3.6:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 80 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 45 } } }, { "case_id": "numbers_versions_dates", "category": "numbers_versions", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 78 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 92 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 0 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 98 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 72 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 78 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 78 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 78 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 95 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 88 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 96 } } }, { "case_id": "simple_list", "category": "lists", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 65 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 35 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 72 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 85 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 55 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 60 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 70 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": false, "score": 75 } } }, { "case_id": "no_change_needed", "category": "no_change_needed", "cells": { "devstral-small-2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma3:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:26b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:e4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gemma4:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "glm-4.7-flash:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "gpt-oss:20b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:30b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "granite4.1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "laguna-xs.2:latest": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "lfm2.5-thinking:1.2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "lfm2:24b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "ministral-3:14b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "ministral-3:3b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "ministral-3:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "nemotron-3-nano:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "nemotron3:33b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo-3.1:32b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "olmo2:7b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 90 }, "qwen3.5:0.8b": { "judge_status": "not_run", "ollama_status": "timeout", "passed": false, "score": 0 }, "qwen3.5:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:2b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:4b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.5:9b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:27b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "qwen3.6:35b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 }, "rnj-1:8b": { "judge_status": "ok", "ollama_status": "ok", "passed": true, "score": 100 } } } ], "model_count": 29, "models": [ { "average_score": 83.67, "case_count": 15, "error_count": 20, "judge_error_count": 20, "median_gpu_busy_percent_avg": 71.2, "median_latency_seconds": 14.6337, "median_power_w_avg": 156.86, "median_vram_mb_peak": 20745.3, "model": "gemma4:26b", "ollama_error_count": 0, "pass_count": 10, "pass_rate": 0.6667, "peak_power_w": 326.0, "peak_vram_mb": 20830.8, "rank": 1, "telemetry_sample_count": 733 }, { "average_score": 82.07, "case_count": 15, "error_count": 14, "judge_error_count": 13, "median_gpu_busy_percent_avg": 37.5, "median_latency_seconds": 108.5076, "median_power_w_avg": 56.24, "median_vram_mb_peak": 23972.7, "model": "qwen3.6:35b", "ollama_error_count": 1, "pass_count": 9, "pass_rate": 0.6, "peak_power_w": 159.0, "peak_vram_mb": 23983.4, "rank": 2, "telemetry_sample_count": 3069 }, { "average_score": 80.73, "case_count": 15, "error_count": 26, "judge_error_count": 26, "median_gpu_busy_percent_avg": 16.5, "median_latency_seconds": 8.0058, "median_power_w_avg": 67.94, "median_vram_mb_peak": 18890.2, "model": "granite4.1:30b", "ollama_error_count": 0, "pass_count": 9, "pass_rate": 0.6, "peak_power_w": 281.0, "peak_vram_mb": 18974.5, "rank": 3, "telemetry_sample_count": 378 }, { "average_score": 77.27, "case_count": 15, "error_count": 21, "judge_error_count": 20, "median_gpu_busy_percent_avg": 86.4, "median_latency_seconds": 58.6964, "median_power_w_avg": 270.14, "median_vram_mb_peak": 10743.3, "model": "qwen3.5:9b", "ollama_error_count": 1, "pass_count": 9, "pass_rate": 0.6, "peak_power_w": 317.0, "peak_vram_mb": 10843.7, "rank": 4, "telemetry_sample_count": 2139 }, { "average_score": 77.2, "case_count": 15, "error_count": 27, "judge_error_count": 27, "median_gpu_busy_percent_avg": 66.7, "median_latency_seconds": 6.4238, "median_power_w_avg": 145.69, "median_vram_mb_peak": 15026.5, "model": "gpt-oss:20b", "ollama_error_count": 0, "pass_count": 6, "pass_rate": 0.4, "peak_power_w": 320.0, "peak_vram_mb": 15110.8, "rank": 5, "telemetry_sample_count": 238 }, { "average_score": 75.53, "case_count": 15, "error_count": 28, "judge_error_count": 28, "median_gpu_busy_percent_avg": 25.2, "median_latency_seconds": 2.2594, "median_power_w_avg": 70.83, "median_vram_mb_peak": 12093.2, "model": "granite4.1:8b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 308.0, "peak_vram_mb": 12096.2, "rank": 6, "telemetry_sample_count": 99 }, { "average_score": 72.27, "case_count": 15, "error_count": 26, "judge_error_count": 25, "median_gpu_busy_percent_avg": 83.6, "median_latency_seconds": 20.3197, "median_power_w_avg": 204.7, "median_vram_mb_peak": 23184.5, "model": "glm-4.7-flash:latest", "ollama_error_count": 1, "pass_count": 8, "pass_rate": 0.5333, "peak_power_w": 292.0, "peak_vram_mb": 23270.0, "rank": 7, "telemetry_sample_count": 1006 }, { "average_score": 72.13, "case_count": 15, "error_count": 28, "judge_error_count": 28, "median_gpu_busy_percent_avg": 37.3, "median_latency_seconds": 36.6626, "median_power_w_avg": 60.65, "median_vram_mb_peak": 23884.5, "model": "nemotron3:33b", "ollama_error_count": 0, "pass_count": 6, "pass_rate": 0.4, "peak_power_w": 235.0, "peak_vram_mb": 23946.4, "rank": 8, "telemetry_sample_count": 1397 }, { "average_score": 72.07, "case_count": 15, "error_count": 32, "judge_error_count": 32, "median_gpu_busy_percent_avg": 59.5, "median_latency_seconds": 8.5258, "median_power_w_avg": 141.5, "median_vram_mb_peak": 11932.0, "model": "gemma4:e4b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 295.0, "peak_vram_mb": 12017.5, "rank": 9, "telemetry_sample_count": 294 }, { "average_score": 71.4, "case_count": 15, "error_count": 21, "judge_error_count": 19, "median_gpu_busy_percent_avg": 94.4, "median_latency_seconds": 57.0776, "median_power_w_avg": 272.93, "median_vram_mb_peak": 23545.3, "model": "olmo-3.1:32b", "ollama_error_count": 2, "pass_count": 8, "pass_rate": 0.5333, "peak_power_w": 315.0, "peak_vram_mb": 23917.7, "rank": 10, "telemetry_sample_count": 2276 }, { "average_score": 71.33, "case_count": 15, "error_count": 28, "judge_error_count": 28, "median_gpu_busy_percent_avg": 86.1, "median_latency_seconds": 68.1213, "median_power_w_avg": 276.03, "median_vram_mb_peak": 8180.5, "model": "qwen3.5:4b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 311.0, "peak_vram_mb": 8266.0, "rank": 11, "telemetry_sample_count": 2189 }, { "average_score": 69.6, "case_count": 15, "error_count": 32, "judge_error_count": 32, "median_gpu_busy_percent_avg": 59.8, "median_latency_seconds": 9.1641, "median_power_w_avg": 109.4, "median_vram_mb_peak": 12576.1, "model": "gemma4:latest", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 278.0, "peak_vram_mb": 12700.1, "rank": 12, "telemetry_sample_count": 286 }, { "average_score": 67.4, "case_count": 15, "error_count": 32, "judge_error_count": 32, "median_gpu_busy_percent_avg": 56.8, "median_latency_seconds": 6.6283, "median_power_w_avg": 117.53, "median_vram_mb_peak": 9134.0, "model": "gemma4:e2b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 312.0, "peak_vram_mb": 9219.0, "rank": 13, "telemetry_sample_count": 232 }, { "average_score": 65.2, "case_count": 15, "error_count": 30, "judge_error_count": 30, "median_gpu_busy_percent_avg": 68.8, "median_latency_seconds": 3.4761, "median_power_w_avg": 173.88, "median_vram_mb_peak": 7925.2, "model": "nemotron-3-nano:4b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 296.0, "peak_vram_mb": 8070.5, "rank": 14, "telemetry_sample_count": 141 }, { "average_score": 65.13, "case_count": 15, "error_count": 30, "judge_error_count": 30, "median_gpu_busy_percent_avg": 57.0, "median_latency_seconds": 10.5886, "median_power_w_avg": 77.35, "median_vram_mb_peak": 22891.1, "model": "devstral-small-2:24b", "ollama_error_count": 0, "pass_count": 6, "pass_rate": 0.4, "peak_power_w": 316.0, "peak_vram_mb": 23007.4, "rank": 15, "telemetry_sample_count": 345 }, { "average_score": 64.13, "case_count": 15, "error_count": 14, "judge_error_count": 10, "median_gpu_busy_percent_avg": 47.7, "median_latency_seconds": 121.96, "median_power_w_avg": 142.69, "median_vram_mb_peak": 24043.6, "model": "qwen3.6:27b", "ollama_error_count": 4, "pass_count": 8, "pass_rate": 0.5333, "peak_power_w": 355.0, "peak_vram_mb": 24049.6, "rank": 16, "telemetry_sample_count": 3769 }, { "average_score": 62.33, "case_count": 15, "error_count": 37, "judge_error_count": 37, "median_gpu_busy_percent_avg": 40.9, "median_latency_seconds": 3.6961, "median_power_w_avg": 69.0, "median_vram_mb_peak": 6703.3, "model": "gemma3:4b", "ollama_error_count": 0, "pass_count": 4, "pass_rate": 0.2667, "peak_power_w": 289.0, "peak_vram_mb": 6866.5, "rank": 17, "telemetry_sample_count": 132 }, { "average_score": 62.2, "case_count": 15, "error_count": 36, "judge_error_count": 36, "median_gpu_busy_percent_avg": 62.4, "median_latency_seconds": 4.2146, "median_power_w_avg": 123.0, "median_vram_mb_peak": 17043.7, "model": "ministral-3:14b", "ollama_error_count": 0, "pass_count": 7, "pass_rate": 0.4667, "peak_power_w": 313.0, "peak_vram_mb": 17114.5, "rank": 18, "telemetry_sample_count": 160 }, { "average_score": 58.6, "case_count": 15, "error_count": 39, "judge_error_count": 39, "median_gpu_busy_percent_avg": 54.9, "median_latency_seconds": 8.0904, "median_power_w_avg": 45.78, "median_vram_mb_peak": 16453.1, "model": "lfm2:24b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 228.0, "peak_vram_mb": 16564.4, "rank": 19, "telemetry_sample_count": 273 }, { "average_score": 58.07, "case_count": 15, "error_count": 37, "judge_error_count": 37, "median_gpu_busy_percent_avg": 31.0, "median_latency_seconds": 1.3766, "median_power_w_avg": 73.75, "median_vram_mb_peak": 6700.7, "model": "granite4.1:3b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 256.0, "peak_vram_mb": 7176.7, "rank": 20, "telemetry_sample_count": 65 }, { "average_score": 56.07, "case_count": 15, "error_count": 37, "judge_error_count": 37, "median_gpu_busy_percent_avg": 59.8, "median_latency_seconds": 3.2377, "median_power_w_avg": 123.0, "median_vram_mb_peak": 12723.6, "model": "ministral-3:8b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 339.0, "peak_vram_mb": 12766.1, "rank": 21, "telemetry_sample_count": 128 }, { "average_score": 55.73, "case_count": 15, "error_count": 36, "judge_error_count": 36, "median_gpu_busy_percent_avg": 51.8, "median_latency_seconds": 2.4815, "median_power_w_avg": 121.0, "median_vram_mb_peak": 11529.1, "model": "rnj-1:8b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 351.0, "peak_vram_mb": 11609.0, "rank": 22, "telemetry_sample_count": 100 }, { "average_score": 54.0, "case_count": 15, "error_count": 38, "judge_error_count": 38, "median_gpu_busy_percent_avg": 26.2, "median_latency_seconds": 1.8999, "median_power_w_avg": 73.2, "median_vram_mb_peak": 8191.5, "model": "olmo2:7b", "ollama_error_count": 0, "pass_count": 5, "pass_rate": 0.3333, "peak_power_w": 305.0, "peak_vram_mb": 8193.0, "rank": 23, "telemetry_sample_count": 86 }, { "average_score": 53.07, "case_count": 15, "error_count": 28, "judge_error_count": 25, "median_gpu_busy_percent_avg": 83.7, "median_latency_seconds": 58.0375, "median_power_w_avg": 236.17, "median_vram_mb_peak": 6013.3, "model": "qwen3.5:2b", "ollama_error_count": 3, "pass_count": 4, "pass_rate": 0.2667, "peak_power_w": 293.0, "peak_vram_mb": 6098.8, "rank": 24, "telemetry_sample_count": 2403 }, { "average_score": 46.47, "case_count": 15, "error_count": 44, "judge_error_count": 44, "median_gpu_busy_percent_avg": 44.0, "median_latency_seconds": 2.4151, "median_power_w_avg": 94.0, "median_vram_mb_peak": 8957.0, "model": "ministral-3:3b", "ollama_error_count": 0, "pass_count": 3, "pass_rate": 0.2, "peak_power_w": 319.0, "peak_vram_mb": 9121.9, "rank": 25, "telemetry_sample_count": 96 }, { "average_score": 40.33, "case_count": 15, "error_count": 42, "judge_error_count": 42, "median_gpu_busy_percent_avg": 49.0, "median_latency_seconds": 10.8602, "median_power_w_avg": 80.58, "median_vram_mb_peak": 24258.1, "model": "laguna-xs.2:latest", "ollama_error_count": 0, "pass_count": 4, "pass_rate": 0.2667, "peak_power_w": 202.0, "peak_vram_mb": 24329.4, "rank": 26, "telemetry_sample_count": 816 }, { "average_score": 24.87, "case_count": 15, "error_count": 49, "judge_error_count": 49, "median_gpu_busy_percent_avg": 72.8, "median_latency_seconds": 5.9512, "median_power_w_avg": 211.38, "median_vram_mb_peak": 3107.6, "model": "lfm2.5-thinking:1.2b", "ollama_error_count": 0, "pass_count": 1, "pass_rate": 0.0667, "peak_power_w": 265.0, "peak_vram_mb": 3179.8, "rank": 27, "telemetry_sample_count": 207 }, { "average_score": 20.0, "case_count": 15, "error_count": 12, "judge_error_count": 0, "median_gpu_busy_percent_avg": 53.9, "median_latency_seconds": 180.0034, "median_power_w_avg": 149.06, "median_vram_mb_peak": 24249.5, "model": "qwen3.5:27b", "ollama_error_count": 12, "pass_count": 3, "pass_rate": 0.2, "peak_power_w": 344.0, "peak_vram_mb": 24293.5, "rank": 28, "telemetry_sample_count": 4664 }, { "average_score": 8.67, "case_count": 15, "error_count": 31, "judge_error_count": 22, "median_gpu_busy_percent_avg": 83.7, "median_latency_seconds": 180.0028, "median_power_w_avg": 204.14, "median_vram_mb_peak": 4432.8, "model": "qwen3.5:0.8b", "ollama_error_count": 9, "pass_count": 0, "pass_rate": 0.0, "peak_power_w": 247.0, "peak_vram_mb": 4534.5, "rank": 29, "telemetry_sample_count": 3697 } ], "result_count": 435, "severity_counts": { "critical": 98, "major": 466, "minor": 311 }, "telemetry": { "available": true, "providers": [ "amd-sysfs" ], "sample_count": 31418 } }, "telemetry": { "device": { "card": "card1", "device_path": "/sys/class/drm/card1/device", "driver": "amdgpu", "pci_address": "0000:0c:00.0", "provider": "amd-sysfs", "reason": "", "status": "ok", "vendor": "0x1002", "vram_total_mb": 24560.0 }, "interval_seconds": 0.5, "mode": "amd-sysfs", "provider": "amd-sysfs", "reason": "", "status": "ok" } }