{
  "experiment": {
    "date": "2026-09-07",
    "title": "Six coding agents, one real feature",
    "task_summary": "Rebuild Bruno's public chat bot so that a price question runs the site's real solar estimator in conversation form: a deterministic bilingual intent gate before the LLM, a one-question-per-message step machine (address via the wizard's geocoder with a numbered pick when ambiguous, owner/renter, monthly kWh or bill with 'no s\u00e9' falling back to the published sample home, savings-vs-backup goal, name, email or WhatsApp), lead creation through the same server libraries the wizard uses, the real roof and pricing analysis, a two-sentence summary with a link to the generated report, a prompt cleanup, an interruption path, a new test file, the repo's publication gate and a real end-to-end conversation against a throwaway server with the private data pack.",
    "repo": "Bruno (Node, ~1,080 tests), private repository",
    "base_commit": "ad1feac",
    "prompt_chars": 6035,
    "grading": "One Fable 5.1 reviewer graded every branch on a 10-item rubric (0-2 each, 20 max) with file-level evidence, re-running the chat test files in each worktree. Self-reports were treated as claims, not evidence.",
    "cost_method": "API-equivalent dollars from each session's own token counters at the provider's list prices. Subscription dollars per task = API dollars x monthly plan fee / monthly API-equivalent allowance, the method of the coding-agent subscription article.",
    "allowances": {
      "claude_max_100": {
        "fee_usd": 100,
        "monthly_api_equiv_usd": 1750,
        "fable_monthly_api_equiv_usd": 1875,
        "source": "One quarter of the measured $7,000 (Opus) / $7,500 (Fable) on the $200 plan"
      },
      "glm_pro_annual": {
        "fee_usd": 56,
        "monthly_api_equiv_usd": 1300,
        "source": "Measured on this account: $250-510 API-equivalent burned per weekly window before the 429 in three windows (Aug 17-19, Aug 31-Sep 3, Sep 4-6, 2026), ~$300/week"
      },
      "codex_pro": {
        "fee_usd": 200,
        "monthly_api_equiv_usd": 7000,
        "source": "Coding-agent subscription article central case"
      }
    },
    "prices_per_million": {
      "claude-fable-5-1": {
        "input": 10,
        "output": 50,
        "cache_read": 0.25,
        "cache_write": 12.5,
        "cache_write_1h": 20
      },
      "claude-opus-5": {
        "input": 5,
        "output": 25,
        "cache_read": 0.5,
        "cache_write": 6.25,
        "cache_write_1h": 10
      },
      "claude-sonnet-5": {
        "input": 2,
        "output": 10,
        "cache_read": 0.2,
        "cache_write": 2.5,
        "cache_write_1h": 4
      },
      "glm-5.3": {
        "input": 1.4,
        "output": 4.4,
        "cache_read": 0.26,
        "cache_write": 1.4,
        "cache_write_1h": null
      },
      "gpt-6-astra": {
        "input": 10,
        "output": 50,
        "cache_read": 1,
        "cache_write": null,
        "cache_write_1h": null
      },
      "claude-opus-4-7": {
        "input": 5,
        "output": 25,
        "cache_read": 0.5,
        "cache_write": 6.25,
        "cache_write_1h": 10
      }
    }
  },
  "runs": [
    {
      "id": "fable-low",
      "label": "Fable 5.1 low",
      "harness": "Claude Code",
      "model": "claude-fable-5-1",
      "effort": "low",
      "plan": "claude_max_100",
      "score": 19.5,
      "wall_min": 29,
      "turns": 199,
      "tool_calls": 129,
      "tokens": {
        "input": 1941,
        "output": 84492,
        "cache_read": 16807464,
        "cache_write": 457291
      },
      "api_usd": 15.850754499999999,
      "sub_usd": 0.8504046142857142,
      "weekly_share_pct": 3.682251979857143,
      "rubric": [
        2,
        2,
        2,
        2,
        1.5,
        2,
        2,
        2,
        2,
        2
      ],
      "notes": [
        "Handled inline addresses and interruptions at every step",
        "Printed en-US number formatting inside Spanish",
        "Pushed the branch to origin unasked"
      ],
      "cost_components": {
        "claude-opus-4-7": {
          "unique_requests": 11,
          "tokens": {
            "input": 21,
            "output": 18598,
            "cache_read": 449199,
            "cache_write": 100959
          },
          "one_hour_cache_write_tokens": 0,
          "api_usd": 1.32064825,
          "sub_usd": 0.0754656142857143
        },
        "claude-fable-5-1": {
          "unique_requests": 90,
          "tokens": {
            "input": 1920,
            "output": 65894,
            "cache_read": 16358265,
            "cache_write": 356332
          },
          "one_hour_cache_write_tokens": 356332,
          "api_usd": 14.53010625,
          "sub_usd": 0.7749389999999999
        }
      }
    },
    {
      "id": "opus-low",
      "label": "Opus 5 low",
      "harness": "Claude Code",
      "model": "claude-opus-5",
      "effort": "low",
      "plan": "claude_max_100",
      "score": 19,
      "wall_min": 29,
      "turns": 262,
      "tool_calls": 192,
      "tokens": {
        "input": 303,
        "output": 97458,
        "cache_read": 23538789,
        "cache_write": 538706
      },
      "api_usd": 18.80562575,
      "sub_usd": 1.0746071857142858,
      "weekly_share_pct": 4.653049114142858,
      "rubric": [
        2,
        2,
        2,
        2,
        2,
        2,
        1.5,
        2,
        1.5,
        2
      ],
      "notes": [
        "Route-faithful wiring, interim 'building your report' message",
        "Found and fixed two bugs in its own live run",
        "No interruption detection at the address step; wrong commit trailer"
      ],
      "cost_components": {
        "claude-opus-4-7": {
          "unique_requests": 21,
          "tokens": {
            "input": 41,
            "output": 31137,
            "cache_read": 986850,
            "cache_write": 210345
          },
          "one_hour_cache_write_tokens": 0,
          "api_usd": 2.58671125,
          "sub_usd": 0.14781207142857145
        },
        "claude-opus-5": {
          "unique_requests": 131,
          "tokens": {
            "input": 262,
            "output": 66321,
            "cache_read": 22551939,
            "cache_write": 328361
          },
          "one_hour_cache_write_tokens": 328361,
          "api_usd": 16.2189145,
          "sub_usd": 0.9267951142857144
        }
      }
    },
    {
      "id": "glm-claude-code",
      "label": "GLM 5.3 via Claude Code",
      "harness": "Claude Code",
      "model": "glm-5.3",
      "effort": "default (thinking on)",
      "plan": "glm_pro_annual",
      "score": 18,
      "wall_min": 41,
      "turns": 349,
      "tool_calls": 204,
      "tokens": {
        "input": 279816,
        "output": 118680,
        "cache_read": 18177344,
        "cache_write": 0
      },
      "api_usd": 5.640043840000001,
      "sub_usd": 0.24295573464615386,
      "weekly_share_pct": 1.878568448246154,
      "rubric": [
        2,
        2,
        2,
        2,
        2,
        2,
        1.5,
        1.5,
        1.5,
        1.5
      ],
      "notes": [
        "Complete prompt cleanup, correct money and sizing",
        "Never stubbed the LLM in tests, so interruption paths were proven only via the canned fallback",
        "Repeated Sonnet's out-of-lane server refactor"
      ],
      "cost_components": {
        "glm-5.3": {
          "tokens": {
            "input": 279816,
            "output": 118680,
            "cache_read": 18177344,
            "cache_write": 0
          },
          "api_usd": 5.640043840000001,
          "sub_usd": 0.24295573464615386,
          "source": "Final modelUsage aggregate"
        }
      }
    },
    {
      "id": "glm-zcode",
      "label": "GLM 5.3 via ZCode",
      "harness": "ZCode",
      "model": "glm-5.3",
      "effort": "default (reasoning: max)",
      "plan": "glm_pro_annual",
      "score": 17.5,
      "wall_min": 36,
      "turns": 268,
      "tool_calls": null,
      "tokens": {
        "input": 143211.0,
        "output": 53346.5,
        "cache_read": 15192384.0,
        "cache_write": 0.0
      },
      "api_usd": 4.3852398400000006,
      "sub_usd": 0.1889026395,
      "weekly_share_pct": 1.4606221947053573,
      "rubric": [
        2,
        1.5,
        2,
        2,
        2,
        1.5,
        1.5,
        2,
        1.5,
        1.5
      ],
      "notes": [
        "Correct USD, es-AR formatting, no second lead",
        "Two sessions worked one tree concurrently and squashed the history",
        "Self-report inflated test counts and stitched the transcript"
      ],
      "cost_components": {
        "glm-5.3": {
          "tokens": {
            "input": 143211.0,
            "output": 53346.5,
            "cache_read": 15192384.0,
            "cache_write": 0.0
          },
          "api_usd": 4.3852398400000006,
          "sub_usd": 0.1889026395,
          "source": "Arithmetic mean of the two provider-recorded session totals",
          "unique_requests": 134.0
        }
      },
      "session_observations": [
        {
          "label": "session 1",
          "unique_requests": 148,
          "tokens": {
            "input": 148577,
            "output": 61172,
            "cache_read": 16778176,
            "cache_write": 0
          },
          "one_hour_cache_write_tokens": 0,
          "api_usd": 4.83949036,
          "sub_usd": 0.208470354,
          "source": "ZCode model_usage database, sess_4ef71c9f (read-only)"
        },
        {
          "label": "session 2",
          "unique_requests": 120,
          "tokens": {
            "input": 137845,
            "output": 45521,
            "cache_read": 13606592,
            "cache_write": 0
          },
          "one_hour_cache_write_tokens": 0,
          "api_usd": 3.93098932,
          "sub_usd": 0.169334925,
          "source": "ZCode model_usage database, sess_d2baabb1 (read-only)"
        }
      ],
      "cost_basis": "mean_per_session",
      "comparison_scope": "Cost and tokens are the arithmetic mean per session. The 8.75/10 grade and recorded wall time describe their shared final branch, not two independently graded attempts."
    },
    {
      "id": "sonnet-medium",
      "label": "Sonnet 5 medium",
      "harness": "Claude Code",
      "model": "claude-sonnet-5",
      "effort": "medium",
      "plan": "claude_max_100",
      "score": 14,
      "wall_min": 41,
      "turns": 407,
      "tool_calls": 252,
      "tokens": {
        "input": 399,
        "output": 142881,
        "cache_read": 45166254,
        "cache_write": 592712
      },
      "api_usd": 13.567119799999999,
      "sub_usd": 0.7752639885714285,
      "weekly_share_pct": 3.356893070514286,
      "rubric": [
        2,
        1,
        2,
        2,
        0.5,
        0.5,
        1.5,
        2,
        1.5,
        1
      ],
      "notes": [
        "Printed an ARS total labelled USD; bill answers never reached sizing",
        "Prompt cleanup half done, installer still named",
        "Live end-to-end run never got past the address step"
      ],
      "cost_components": {
        "claude-opus-4-7": {
          "unique_requests": 15,
          "tokens": {
            "input": 35,
            "output": 16316,
            "cache_read": 719560,
            "cache_write": 121200
          },
          "one_hour_cache_write_tokens": 0,
          "api_usd": 1.525355,
          "sub_usd": 0.08716314285714287
        },
        "claude-sonnet-5": {
          "unique_requests": 182,
          "tokens": {
            "input": 364,
            "output": 126565,
            "cache_read": 44446694,
            "cache_write": 471512
          },
          "one_hour_cache_write_tokens": 471512,
          "api_usd": 12.0417648,
          "sub_usd": 0.6881008457142856
        }
      }
    },
    {
      "id": "codex-astra-low",
      "label": "Codex GPT-6 Astra low",
      "harness": "Codex CLI",
      "model": "gpt-6-astra",
      "effort": "low",
      "plan": "codex_pro",
      "score": 15.5,
      "wall_min": 14,
      "turns": null,
      "tool_calls": 107,
      "tokens": {
        "input": 105830,
        "output": 18309,
        "cache_read": 4079104,
        "cache_write": 0
      },
      "api_usd": 6.052854,
      "sub_usd": 0.17293868571428572,
      "weekly_share_pct": 0.37441225457142857,
      "rubric": [
        2,
        1.5,
        1.5,
        2,
        2,
        1.5,
        1.5,
        1.5,
        1,
        1
      ],
      "notes": [
        "Fastest run and cleanest lane: correct USD, es-AR formatting, an output guard, a per-thread lock",
        "Parsers too strict, so ordinary Spanish answers fell through to the LLM as interruptions",
        "Stopped at the geocoder wall the others routed around, so no live report link; reported a symlink it created as pre-existing"
      ],
      "cost_components": {
        "gpt-6-astra": {
          "unique_requests": null,
          "tokens": {
            "input": 105830,
            "output": 18309,
            "cache_read": 4079104,
            "cache_write": 0
          },
          "one_hour_cache_write_tokens": 0,
          "api_usd": 6.052854,
          "sub_usd": 0.17293868571428572,
          "source": "Codex turn.completed cumulative usage, counted once; fresh input = input \u2212 cached"
        }
      }
    }
  ],
  "rubric_items": [
    "Intent gate catches all 7 openers in both languages",
    "Step machine, parsers, skips known data",
    "Reuses the wizard's geocoder, numbered pick of at most 3",
    "'no s\u00e9' falls back to the published sample size",
    "Lead and report through the real libraries, correct money",
    "Prompt cleanup: no price dump, no early visit, 'nosotros'",
    "Interruption answered, then the pending question repeated",
    "Tests: openers, happy path, unknown consumption, interruption, ambiguity",
    "Gate green, lane respected, trailer, not merged",
    "Real end-to-end transcript and an honest report"
  ],
  "pending": []
}