{ "model": "TAARM", "version": "1.0.0", "method": "Synthetic Bernoulli fault-injection validation; not live provider benchmarks", "runsPerCase": 100000, "rows": [ { "name": "fourteen-independent", "tree": { "id": "root", "type": "serial", "name": "Workflow", "branch": 0.5, "children": [ { "id": "llm-1", "name": "llm-1", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "llm-2", "name": "llm-2", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "llm-3", "name": "llm-3", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "llm-4", "name": "llm-4", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "llm-5", "name": "llm-5", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "llm-6", "name": "llm-6", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "llm-7", "name": "llm-7", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "llm-8", "name": "llm-8", "type": "step", "kind": "llm", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 }, { "id": "api-9", "name": "api-9", "type": "step", "kind": "api", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.001, "ms": 150, "backoff": 500 }, { "id": "api-10", "name": "api-10", "type": "step", "kind": "api", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.001, "ms": 150, "backoff": 500 }, { "id": "api-11", "name": "api-11", "type": "step", "kind": "api", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.001, "ms": 150, "backoff": 500 }, { "id": "database-12", "name": "database-12", "type": "step", "kind": "database", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.001, "ms": 150, "backoff": 500 }, { "id": "database-13", "name": "database-13", "type": "step", "kind": "database", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.001, "ms": 150, "backoff": 500 }, { "id": "image-14", "name": "image-14", "type": "step", "kind": "image", "p": 0.985, "q": 0, "attempts": 1, "cost": 0.04, "ms": 10000, "backoff": 500 } ] }, "outage": 0, "expected": { "success": 0.8092956889835273, "attempts": 12.713620734431503, "cost": 0.11308809937270238, "external": 11.032654280465371 }, "observed": { "runs": 100000, "seed": 20260921, "successes": 80711, "success": 0.80711, "attempts": 12.70677, "cost": 0.11300649000000994, "external": 11.02791, "interval": [ 0.8046527122043973, 0.8095436936935919 ], "p50ms": 20350, "p95ms": 20350 } }, { "name": "serial", "tree": { "id": "g", "type": "serial", "name": "serial", "children": [ { "id": "a", "name": "a", "type": "step", "kind": "llm", "p": 0.8, "q": 0, "attempts": 3, "cost": 2, "ms": 10, "backoff": 5 }, { "id": "b", "name": "b", "type": "step", "kind": "llm", "p": 0.5, "q": 0, "attempts": 1, "cost": 1, "ms": 1200, "backoff": 500 } ], "branch": 0.3 }, "outage": 0, "expected": { "success": 0.496, "attempts": 2.232, "cost": 3.472, "external": 2.232 }, "observed": { "runs": 100000, "seed": 20260922, "successes": 49298, "success": 0.49298, "attempts": 2.23079, "cost": 3.46989, "external": 2.23079, "interval": [ 0.4898816594546642, 0.49607887986543653 ], "p50ms": 1210, "p95ms": 1225 } }, { "name": "parallel-all", "tree": { "id": "g", "type": "parallel", "name": "parallel", "children": [ { "id": "a", "name": "a", "type": "step", "kind": "llm", "p": 0.8, "q": 0, "attempts": 3, "cost": 2, "ms": 10, "backoff": 5 }, { "id": "b", "name": "b", "type": "step", "kind": "llm", "p": 0.5, "q": 0, "attempts": 1, "cost": 1, "ms": 1200, "backoff": 500 } ], "branch": 0.3 }, "outage": 0, "expected": { "success": 0.496, "attempts": 2.24, "cost": 3.48, "external": 2.24 }, "observed": { "runs": 100000, "seed": 20260923, "successes": 49586, "success": 0.49586, "attempts": 2.24002, "cost": 3.48004, "external": 2.24002, "interval": [ 0.49276134961606494, 0.49895896844450743 ], "p50ms": 1200, "p95ms": 1200 } }, { "name": "fallback", "tree": { "id": "g", "type": "fallback", "name": "fallback", "children": [ { "id": "a", "name": "a", "type": "step", "kind": "llm", "p": 0.8, "q": 0, "attempts": 3, "cost": 2, "ms": 10, "backoff": 5 }, { "id": "b", "name": "b", "type": "step", "kind": "llm", "p": 0.5, "q": 0, "attempts": 1, "cost": 1, "ms": 1200, "backoff": 500 } ], "branch": 0.3 }, "outage": 0, "expected": { "success": 0.996, "attempts": 1.248, "cost": 2.488, "external": 1.248 }, "observed": { "runs": 100000, "seed": 20260924, "successes": 99612, "success": 0.99612, "attempts": 1.24665, "cost": 2.48562, "external": 1.24665, "interval": [ 0.9957151604506147, 0.9964867245225559 ], "p50ms": 10, "p95ms": 25 } }, { "name": "conditional", "tree": { "id": "g", "type": "conditional", "name": "conditional", "children": [ { "id": "a", "name": "a", "type": "step", "kind": "llm", "p": 0.8, "q": 0, "attempts": 3, "cost": 2, "ms": 10, "backoff": 5 }, { "id": "b", "name": "b", "type": "step", "kind": "llm", "p": 0.5, "q": 0, "attempts": 1, "cost": 1, "ms": 1200, "backoff": 500 } ], "branch": 0.3 }, "outage": 0, "expected": { "success": 0.6476, "attempts": 1.072, "cost": 1.444, "external": 1.072 }, "observed": { "runs": 100000, "seed": 20260925, "successes": 65046, "success": 0.65046, "attempts": 1.07205, "cost": 1.44737, "external": 1.07205, "interval": [ 0.6474989357102777, 0.6534095050158838 ], "p50ms": 1200, "p95ms": 1200 } }, { "name": "persistent", "tree": { "id": "a", "name": "a", "type": "step", "kind": "llm", "p": 0.8, "q": 0.1, "attempts": 3, "cost": 2, "ms": 10, "backoff": 5 }, "outage": 0, "expected": { "success": 0.8928, "attempts": 1.4160000000000001, "cost": 2.8320000000000003, "external": 1.4160000000000001 }, "observed": { "runs": 100000, "seed": 20260926, "successes": 89250, "success": 0.8925, "attempts": 1.41775, "cost": 2.8355, "external": 1.41775, "interval": [ 0.8905650985408436, 0.8944047471657787 ], "p50ms": 10, "p95ms": 40 } }, { "name": "preflight", "tree": { "id": "g", "type": "fallback", "name": "fallback", "children": [ { "id": "a", "name": "a", "type": "step", "kind": "llm", "p": 0.8, "q": 0, "attempts": 3, "cost": 2, "ms": 10, "backoff": 5 }, { "id": "b", "name": "b", "type": "step", "kind": "llm", "p": 0.5, "q": 0, "attempts": 1, "cost": 1, "ms": 1200, "backoff": 500 } ], "branch": 0.3 }, "outage": 0.1, "expected": { "success": 0.8964, "attempts": 1.1232, "cost": 2.2392, "external": 1.1232 }, "observed": { "runs": 100000, "seed": 20260927, "successes": 89606, "success": 0.89606, "attempts": 1.12411, "cost": 2.24081, "external": 1.12411, "interval": [ 0.8941532540737676, 0.8979363181314932 ], "p50ms": 10, "p95ms": 25 } }, { "name": "nested", "tree": { "id": "g", "type": "serial", "name": "serial", "children": [ { "id": "inner", "type": "conditional", "name": "conditional", "children": [ { "id": "a", "name": "a", "type": "step", "kind": "llm", "p": 0.8, "q": 0, "attempts": 3, "cost": 2, "ms": 10, "backoff": 5 }, { "id": "b", "name": "b", "type": "step", "kind": "llm", "p": 0.5, "q": 0, "attempts": 1, "cost": 1, "ms": 1200, "backoff": 500 } ], "branch": 0.3 }, { "id": "c", "name": "c", "type": "step", "kind": "llm", "p": 0.9, "q": 0, "attempts": 1, "cost": 0.01, "ms": 1200, "backoff": 500 } ], "branch": 0.3 }, "outage": 0.02, "expected": { "success": 0.5711832, "attempts": 1.685208, "cost": 1.4214664799999999, "external": 1.685208 }, "observed": { "runs": 100000, "seed": 20260928, "successes": 57041, "success": 0.57041, "attempts": 1.68472, "cost": 1.4215203999991226, "external": 1.68472, "interval": [ 0.5673392585273215, 0.5734753321381646 ], "p50ms": 1210, "p95ms": 2400 } } ] }