{
  "title": "Six of our agent's seventeen tools had never run.",
  "url": "https://cybiqon.in/lab/six-of-our-agents-tools-had-never-run",
  "author": "Prajjwal Pathak",
  "published": "2026-08-12",
  "license": {
    "name": "CC BY 4.0",
    "url": "https://creativecommons.org/licenses/by/4.0/"
  },
  "citation": "Prajjwal Pathak, “Six of our agent's seventeen tools had never run.”, Cybiqon Lab, 2026-08-12. https://cybiqon.in/lab/six-of-our-agents-tools-had-never-run",
  "tables": [
    {
      "id": "what-an-agent-evaluation-actually-has-to-measure",
      "section": "What an agent evaluation actually has to measure",
      "csv": "https://cybiqon.in/data/lab/six-of-our-agents-tools-had-never-run/what-an-agent-evaluation-actually-has-to-measure.csv",
      "columns": [
        "Dimension",
        "What it catches"
      ],
      "rows": [
        [
          "Tool selection",
          "Right tool, no unnecessary ones. Precision and recall against a per-scenario allow-list"
        ],
        [
          "Tool arguments",
          "The user said \"in Bangalore\" and the query kept it"
        ],
        [
          "Trajectory",
          "Ordering, duplicate calls, termination, and silent argument retries"
        ],
        [
          "Skill discovery",
          "Did it load the right playbook, before acting on it"
        ],
        [
          "Task quality",
          "An LLM judge against a per-scenario rubric, 0–4"
        ],
        [
          "Guardrails",
          "Prompt injection, direct and indirect"
        ],
        [
          "Approval safety",
          "Did it pause instead of spending, and did zero paid calls actually fire"
        ],
        [
          "Cost and latency",
          "Per scenario, priced from our own rate table"
        ]
      ]
    },
    {
      "id": "what-123-runs-found",
      "section": "What 123 runs found",
      "csv": "https://cybiqon.in/data/lab/six-of-our-agents-tools-had-never-run/what-123-runs-found.csv",
      "columns": [
        "What we measured",
        "Score",
        "Reading"
      ],
      "rows": [
        [
          "Spend safety — never charges without asking",
          "1.00",
          "Perfect across 123 runs"
        ],
        [
          "Security — resists prompt injection",
          "1.00",
          "Including attacks hidden in LinkedIn posts"
        ],
        [
          "Skill discovery — finds the right playbook",
          "1.00",
          ""
        ],
        [
          "Tool arguments",
          "1.00",
          ""
        ],
        [
          "Sequencing — no loops or repeats",
          "1.00",
          ""
        ],
        [
          "Picks the right tool",
          "0.98",
          ""
        ],
        [
          "Answer quality (graded 0–4)",
          "3.74",
          "92% pass rate"
        ]
      ]
    }
  ]
}