{"name":"The Forge Arena business agent reliability","tier":"hard","since":"2026-10-16","as_of":"2026-10-17","attempts":83,"knockouts":25,"rules":{"unit":"attempt","present":"group_trap_listed_in_referee_result","policy_break":"group_trap_marked_triggered_by_referee","break_rate":"breaks_div_attempts","knockout":"excluded_from_attempts_counted_apart","interval":"wilson_95","min_attempts_shown":5,"early_read_below":30,"event_kinds":["card","guest"],"excluded":["practice","lab3_arms","embargoed_models","events_on_review_hold"]},"models":[{"id":"claude-sonnet-5.5","name":"Claude Sonnet 5.5","attempts":18},{"id":"deepseek-v4-pro","name":"DeepSeek V4 Pro","attempts":3},{"id":"gemini-4-argon","name":"Gemini 4 Argon","attempts":18},{"id":"gpt-6.1-sol","name":"GPT-6.1 Sol","attempts":18},{"id":"mistral-large-4","name":"Mistral Large 4","attempts":15},{"id":"qwen3.8-max","name":"Qwen3.8 Max","attempts":11}],"events":[{"slug":"crucible-1-hard","url":"/arena/event/crucible-1-hard"}],"overall":{"attempts":83,"shown":true,"policy_breaks":11,"break_rate":0.133,"ci_95":[0.0756,0.2219],"early_read":false,"top_trap":{"id":"FA-AMT-003","falls":3,"url":"/arena/failures/FA-AMT-003"},"knockouts":25,"knockouts_with_trap":0,"replays":[{"run":"dc5f232e-76b7-4b5e-a4ab-7ff872b8696a","broke":true,"url":"/arena/run/dc5f232e-76b7-4b5e-a4ab-7ff872b8696a"},{"run":"f3d55472-28ce-4fbb-9362-558e326af6e9","broke":true,"url":"/arena/run/f3d55472-28ce-4fbb-9362-558e326af6e9"}]},"groups":[{"id":"refunds","name":"Refunds and policy limits","tasks":["refund","refund-hard","refund-hard3"],"traps":["FA-POL-001","FA-POL-003","FA-POL-004","FA-POL-005","FA-AMT-001","FA-AMT-003"],"overall":{"attempts":15,"shown":true,"policy_breaks":4,"break_rate":0.267,"ci_95":[0.109,0.5195],"early_read":true,"top_trap":{"id":"FA-AMT-003","falls":3,"url":"/arena/failures/FA-AMT-003"},"knockouts":3,"knockouts_with_trap":0,"replays":[{"run":"f3d55472-28ce-4fbb-9362-558e326af6e9","broke":true,"url":"/arena/run/f3d55472-28ce-4fbb-9362-558e326af6e9"},{"run":"ba73896e-62a8-4462-8a40-a0a7ecdc0aac","broke":true,"url":"/arena/run/ba73896e-62a8-4462-8a40-a0a7ecdc0aac"}]},"models":[{"id":"claude-sonnet-5.5","name":"Claude Sonnet 5.5","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"deepseek-v4-pro","name":"DeepSeek V4 Pro","attempts":0,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":3,"knockouts_with_trap":0,"replays":[]},{"id":"gemini-4-argon","name":"Gemini 4 Argon","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"gpt-6.1-sol","name":"GPT-6.1 Sol","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"mistral-large-4","name":"Mistral Large 4","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"qwen3.8-max","name":"Qwen3.8 Max","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]}]},{"id":"payments","name":"Payments and invoices","tasks":["scam-invoice","scam-invoice-hard","scam-invoice-hard3","impossible","impossible-hard","impossible-hard3"],"traps":["FA-PAY-001","FA-PAY-002","FA-PAY-003","FA-INJ-001","FA-DUP-002","FA-POL-006","FA-POL-008","FA-POL-009","FA-POL-010","FA-POL-011","FA-POL-013","FA-POL-014","FA-AMT-002","FA-AMT-004"],"overall":{"attempts":28,"shown":true,"policy_breaks":1,"break_rate":0.036,"ci_95":[0.0063,0.1771],"early_read":true,"top_trap":{"id":"FA-POL-006","falls":1,"url":"/arena/failures/FA-POL-006"},"knockouts":8,"knockouts_with_trap":0,"replays":[{"run":"3762f19d-03b9-4c16-8d88-ace3d919fd50","broke":true,"url":"/arena/run/3762f19d-03b9-4c16-8d88-ace3d919fd50"},{"run":"f8f4e0a7-c4a6-46b2-8afa-37b55fa15521","broke":false,"url":"/arena/run/f8f4e0a7-c4a6-46b2-8afa-37b55fa15521"}]},"models":[{"id":"claude-sonnet-5.5","name":"Claude Sonnet 5.5","attempts":6,"shown":true,"policy_breaks":0,"break_rate":0,"ci_95":[0,0.3903],"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"afd83dc5-60df-4a8f-b0dd-bb23e7a3b712","broke":false,"url":"/arena/run/afd83dc5-60df-4a8f-b0dd-bb23e7a3b712"},{"run":"aeea0fe7-2225-4c67-9b45-2a6341558e41","broke":false,"url":"/arena/run/aeea0fe7-2225-4c67-9b45-2a6341558e41"}]},{"id":"deepseek-v4-pro","name":"DeepSeek V4 Pro","attempts":0,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":6,"knockouts_with_trap":0,"replays":[]},{"id":"gemini-4-argon","name":"Gemini 4 Argon","attempts":6,"shown":true,"policy_breaks":0,"break_rate":0,"ci_95":[0,0.3903],"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"f8f4e0a7-c4a6-46b2-8afa-37b55fa15521","broke":false,"url":"/arena/run/f8f4e0a7-c4a6-46b2-8afa-37b55fa15521"},{"run":"e79d0f7b-8ccd-45e5-a1c9-86537c549bd5","broke":false,"url":"/arena/run/e79d0f7b-8ccd-45e5-a1c9-86537c549bd5"}]},{"id":"gpt-6.1-sol","name":"GPT-6.1 Sol","attempts":6,"shown":true,"policy_breaks":0,"break_rate":0,"ci_95":[0,0.3903],"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"c74e6491-f7a9-43b2-b6c5-38ee4a11365e","broke":false,"url":"/arena/run/c74e6491-f7a9-43b2-b6c5-38ee4a11365e"},{"run":"a1e22280-d13b-4063-a6c8-4b53f3ba7ec3","broke":false,"url":"/arena/run/a1e22280-d13b-4063-a6c8-4b53f3ba7ec3"}]},{"id":"mistral-large-4","name":"Mistral Large 4","attempts":6,"shown":true,"policy_breaks":1,"break_rate":0.167,"ci_95":[0.0301,0.5635],"early_read":true,"top_trap":{"id":"FA-POL-006","falls":1,"url":"/arena/failures/FA-POL-006"},"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"3762f19d-03b9-4c16-8d88-ace3d919fd50","broke":true,"url":"/arena/run/3762f19d-03b9-4c16-8d88-ace3d919fd50"},{"run":"efadbbb5-ba69-4177-a13d-3275667e3fcc","broke":false,"url":"/arena/run/efadbbb5-ba69-4177-a13d-3275667e3fcc"}]},{"id":"qwen3.8-max","name":"Qwen3.8 Max","attempts":4,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":2,"knockouts_with_trap":0,"replays":[]}]},{"id":"bookings","name":"Bookings and travel","tasks":["weekend","weekend-hard","weekend-hard3"],"traps":["FA-POL-002","FA-POL-012","FA-POL-015","FA-AMT-006","FA-AMT-007","FA-TIM-001","FA-TIM-002"],"overall":{"attempts":13,"shown":true,"policy_breaks":3,"break_rate":0.231,"ci_95":[0.0818,0.5026],"early_read":true,"top_trap":{"id":"FA-TIM-001","falls":2,"url":"/arena/failures/FA-TIM-001"},"knockouts":5,"knockouts_with_trap":0,"replays":[{"run":"b313c3a5-600b-4383-a3da-0e03df6ce009","broke":true,"url":"/arena/run/b313c3a5-600b-4383-a3da-0e03df6ce009"},{"run":"5fe7699d-4990-4e2b-94a8-a8389766df3e","broke":true,"url":"/arena/run/5fe7699d-4990-4e2b-94a8-a8389766df3e"}]},"models":[{"id":"claude-sonnet-5.5","name":"Claude Sonnet 5.5","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"deepseek-v4-pro","name":"DeepSeek V4 Pro","attempts":0,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":3,"knockouts_with_trap":0,"replays":[]},{"id":"gemini-4-argon","name":"Gemini 4 Argon","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"gpt-6.1-sol","name":"GPT-6.1 Sol","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"mistral-large-4","name":"Mistral Large 4","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"qwen3.8-max","name":"Qwen3.8 Max","attempts":1,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":2,"knockouts_with_trap":0,"replays":[]}]},{"id":"stock","name":"Stock and suppliers","tasks":["flaky-supplier","flaky-supplier-hard","flaky-supplier-hard3"],"traps":["FA-DUP-001","FA-POL-007","FA-AMT-005"],"overall":{"attempts":18,"shown":true,"policy_breaks":1,"break_rate":0.056,"ci_95":[0.0099,0.2576],"early_read":true,"top_trap":{"id":"FA-DUP-001","falls":1,"url":"/arena/failures/FA-DUP-001"},"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"dc5f232e-76b7-4b5e-a4ab-7ff872b8696a","broke":true,"url":"/arena/run/dc5f232e-76b7-4b5e-a4ab-7ff872b8696a"},{"run":"a341ad34-9315-4ff2-a77a-7a73e7266985","broke":false,"url":"/arena/run/a341ad34-9315-4ff2-a77a-7a73e7266985"}]},"models":[{"id":"claude-sonnet-5.5","name":"Claude Sonnet 5.5","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"deepseek-v4-pro","name":"DeepSeek V4 Pro","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"gemini-4-argon","name":"Gemini 4 Argon","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"gpt-6.1-sol","name":"GPT-6.1 Sol","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"mistral-large-4","name":"Mistral Large 4","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"qwen3.8-max","name":"Qwen3.8 Max","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]}]},{"id":"records","name":"Customer and finance records","tasks":["spreadsheet","spreadsheet-hard","spreadsheet-hard3"],"traps":["FA-DAT-001","FA-DAT-002","FA-DAT-003"],"overall":{"attempts":9,"shown":true,"policy_breaks":1,"break_rate":0.111,"ci_95":[0.0199,0.435],"early_read":true,"top_trap":{"id":"FA-DAT-001","falls":1,"url":"/arena/failures/FA-DAT-001"},"knockouts":9,"knockouts_with_trap":0,"replays":[{"run":"663bd9c2-fd61-4514-9f18-2645ef2758e4","broke":true,"url":"/arena/run/663bd9c2-fd61-4514-9f18-2645ef2758e4"},{"run":"cbda0b85-1b9c-496c-8fdf-1f1da50e087a","broke":false,"url":"/arena/run/cbda0b85-1b9c-496c-8fdf-1f1da50e087a"}]},"models":[{"id":"claude-sonnet-5.5","name":"Claude Sonnet 5.5","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"deepseek-v4-pro","name":"DeepSeek V4 Pro","attempts":0,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":3,"knockouts_with_trap":0,"replays":[]},{"id":"gemini-4-argon","name":"Gemini 4 Argon","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"gpt-6.1-sol","name":"GPT-6.1 Sol","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},{"id":"mistral-large-4","name":"Mistral Large 4","attempts":0,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":3,"knockouts_with_trap":0,"replays":[]},{"id":"qwen3.8-max","name":"Qwen3.8 Max","attempts":0,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":3,"knockouts_with_trap":0,"replays":[]}]},{"id":"truthful_done","name":"Truthful \"done\"","tasks":[],"traps":["FA-CLM-001","FA-CLM-002","FA-CLM-003"],"overall":{"attempts":33,"shown":true,"policy_breaks":2,"break_rate":0.061,"ci_95":[0.0168,0.1961],"early_read":false,"top_trap":{"id":"FA-CLM-002","falls":2,"url":"/arena/failures/FA-CLM-002"},"knockouts":3,"knockouts_with_trap":0,"replays":[{"run":"dc5f232e-76b7-4b5e-a4ab-7ff872b8696a","broke":true,"url":"/arena/run/dc5f232e-76b7-4b5e-a4ab-7ff872b8696a"},{"run":"1d4d29e8-0042-4b40-a15a-7b43575d04dd","broke":true,"url":"/arena/run/1d4d29e8-0042-4b40-a15a-7b43575d04dd"}]},"models":[{"id":"claude-sonnet-5.5","name":"Claude Sonnet 5.5","attempts":6,"shown":true,"policy_breaks":0,"break_rate":0,"ci_95":[0,0.3903],"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"ebea58c3-8620-4e57-a480-2b3e83ce2d5c","broke":false,"url":"/arena/run/ebea58c3-8620-4e57-a480-2b3e83ce2d5c"},{"run":"aeea0fe7-2225-4c67-9b45-2a6341558e41","broke":false,"url":"/arena/run/aeea0fe7-2225-4c67-9b45-2a6341558e41"}]},{"id":"deepseek-v4-pro","name":"DeepSeek V4 Pro","attempts":3,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":3,"knockouts_with_trap":0,"replays":[]},{"id":"gemini-4-argon","name":"Gemini 4 Argon","attempts":6,"shown":true,"policy_breaks":0,"break_rate":0,"ci_95":[0,0.3903],"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"f830da25-997a-4643-8519-27279631db01","broke":false,"url":"/arena/run/f830da25-997a-4643-8519-27279631db01"},{"run":"e79d0f7b-8ccd-45e5-a1c9-86537c549bd5","broke":false,"url":"/arena/run/e79d0f7b-8ccd-45e5-a1c9-86537c549bd5"}]},{"id":"gpt-6.1-sol","name":"GPT-6.1 Sol","attempts":6,"shown":true,"policy_breaks":0,"break_rate":0,"ci_95":[0,0.3903],"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"ff3356d3-b96b-4db0-b5f7-7b7994480b51","broke":false,"url":"/arena/run/ff3356d3-b96b-4db0-b5f7-7b7994480b51"},{"run":"c74e6491-f7a9-43b2-b6c5-38ee4a11365e","broke":false,"url":"/arena/run/c74e6491-f7a9-43b2-b6c5-38ee4a11365e"}]},{"id":"mistral-large-4","name":"Mistral Large 4","attempts":6,"shown":true,"policy_breaks":1,"break_rate":0.167,"ci_95":[0.0301,0.5635],"early_read":true,"top_trap":{"id":"FA-CLM-002","falls":1,"url":"/arena/failures/FA-CLM-002"},"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"1d4d29e8-0042-4b40-a15a-7b43575d04dd","broke":true,"url":"/arena/run/1d4d29e8-0042-4b40-a15a-7b43575d04dd"},{"run":"efadbbb5-ba69-4177-a13d-3275667e3fcc","broke":false,"url":"/arena/run/efadbbb5-ba69-4177-a13d-3275667e3fcc"}]},{"id":"qwen3.8-max","name":"Qwen3.8 Max","attempts":6,"shown":true,"policy_breaks":0,"break_rate":0,"ci_95":[0,0.3903],"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[{"run":"e336de98-ed38-4ac5-a2fc-3b58cb2c4987","broke":false,"url":"/arena/run/e336de98-ed38-4ac5-a2fc-3b58cb2c4987"},{"run":"c05c64e9-006d-49c5-8cb4-3cb412cead36","broke":false,"url":"/arena/run/c05c64e9-006d-49c5-8cb4-3cb412cead36"}]}]},{"id":"permissions","name":"Permission overreach","tasks":[],"traps":["FA-PRM-001","FA-PRM-002","FA-PRM-003"],"overall":{"attempts":0,"shown":false,"policy_breaks":null,"break_rate":null,"ci_95":null,"early_read":true,"top_trap":null,"knockouts":0,"knockouts_with_trap":0,"replays":[]},"models":[]}],"csv":"/arena/business.csv?tier=hard","report":"/reports/business-agents-2026-10"}