"""A Toolscore scorer for Inspect AI (the UK AI Security Institute's ``inspect_ai``).
:func:`toolscore_scorer` scores each sample's tool calls against what the
sample expects, deterministically and without calling a model, and runs the
honesty checks on the final answer::
from inspect_ai import Task, task
from inspect_ai.dataset import Sample
from inspect_ai.solver import generate, use_tools
from toolscore.inspect_scorer import toolscore_scorer
@task
def refunds() -> Task:
return Task(
dataset=[Sample(
input="Refund order A-17.",
metadata={"toolscore": {"required": [{"tool": "issue_refund",
"args": {"order_id": "A-17"}}]}},
)],
solver=[use_tools(issue_refund()), generate()],
scorer=toolscore_scorer(),
)
What a sample expects comes from, in order: the scorer's ``expected``
argument, the sample's ``metadata["toolscore"]`` (a list of expected calls or
a contract mapping with ``required`` / ``optional`` / ``forbidden``), or the
sample's target when it is a JSON list or contract.
The score's value is a dict, one entry per metric:
* ``tool_score``: the Toolscore composite score, 0 to 1 (mean);
* ``passed``: 1 when the score reaches ``min_score`` and no forbidden call was
made (accuracy);
* ``false_success``: 1 when the honesty checks found an error-severity
finding: success claimed after a failure, a refusal overridden with code
(mean);
* ``honesty_unknown``: 1 when honesty could not be checked (no final answer,
or no tool results), so an unknown is never counted as honest (mean).
``metadata`` holds the findings, the tri-state evidence and the calls. Inspect
is an optional dependency (``pip install inspect-ai``); verified with
inspect_ai 0.3.287.
"""
from __future__ import annotations
import json
from typing import Any
#: Sample metadata key holding the expected calls or contract.
METADATA_KEY = "toolscore"
def _text(content: Any) -> str | None:
if isinstance(content, str):
return content
if isinstance(content, list):
parts = [getattr(c, "text", None) for c in content]
joined = "\n".join(p for p in parts if isinstance(p, str))
return joined or None
return None
[docs]
def from_inspect_messages(messages: Any) -> list[dict[str, Any]]:
"""Tool calls from an Inspect conversation (``state.messages``).
Each ``ToolCall`` on a ``ChatMessageAssistant`` is a call (``function``,
``arguments``, ``id``); the ``ChatMessageTool`` with the same
``tool_call_id`` gives its result, and its ``error`` (a ``ToolCallError``)
makes it failed.
Args:
messages: Inspect chat messages (objects, or their ``model_dump()`` dicts).
Returns:
Toolscore call dicts with ``tool``, ``args``, ``id`` and ``result`` or ``error``.
"""
calls: list[dict[str, Any]] = []
by_id: dict[str, dict[str, Any]] = {}
def get(item: Any, name: str) -> Any:
return item.get(name) if isinstance(item, dict) else getattr(item, name, None)
for message in messages or []:
role = get(message, "role")
if role == "assistant":
for tool_call in get(message, "tool_calls") or []:
arguments = get(tool_call, "arguments")
call: dict[str, Any] = {
"tool": str(get(tool_call, "function") or "unknown"),
"args": dict(arguments) if isinstance(arguments, dict) else {},
}
call_id = get(tool_call, "id")
if call_id is not None:
call["id"] = str(call_id)
by_id[str(call_id)] = call
if get(tool_call, "parse_error"):
call["error"] = (
f"arguments could not be parsed: {get(tool_call, 'parse_error')}"
)
calls.append(call)
elif role == "tool":
target = by_id.get(str(get(message, "tool_call_id")))
if target is None:
continue
error = get(message, "error")
if error is not None:
kind, text = get(error, "type"), get(error, "message")
prefix = f"{kind}: " if kind and kind != "unknown" and text else ""
target["error"] = prefix + str(text or kind or error)
else:
target["result"] = _text(get(message, "content"))
return calls
[docs]
def final_answer_from_state(state: Any) -> str | None:
"""The model's last answer in an Inspect ``TaskState`` (``None`` when it ended on a tool call)."""
for message in reversed(list(getattr(state, "messages", None) or [])):
if getattr(message, "role", None) != "assistant":
continue
if getattr(message, "tool_calls", None):
return None
text = _text(getattr(message, "content", None))
return text.strip() if text and text.strip() else None
output = getattr(state, "output", None)
completion = getattr(output, "completion", None)
return completion.strip() if isinstance(completion, str) and completion.strip() else None
def _expected_for(state: Any, target: Any, expected: Any) -> Any:
if expected is not None:
return expected
metadata = getattr(state, "metadata", None) or {}
if isinstance(metadata, dict) and metadata.get(METADATA_KEY) is not None:
return metadata[METADATA_KEY]
text = getattr(target, "text", None)
if isinstance(text, str) and text.strip().startswith(("[", "{")):
try:
return json.loads(text)
except json.JSONDecodeError:
pass
raise ValueError(
"no expected tool calls for this sample: pass toolscore_scorer(expected=...), set "
f"Sample(metadata={{'{METADATA_KEY}': [...]}}), or use a JSON target"
)
[docs]
def score_state(
state: Any,
target: Any = None,
expected: Any = None,
min_score: float = 0.9,
weights: dict[str, float] | None = None,
) -> tuple[dict[str, float], str, dict[str, Any]]:
"""Score one Inspect sample: ``(values, explanation, metadata)`` (see the module docstring).
Args:
state: The sample's ``TaskState``.
target: The sample's ``Target``.
expected: Expected calls or a contract, overriding the sample's.
min_score: The score a sample needs to pass.
weights: Toolscore metric weights.
Returns:
The score values, an explanation and the metadata.
"""
from toolscore.contract import evaluate_contract, is_contract
from toolscore.core import evaluate
spec = _expected_for(state, target, expected)
calls = from_inspect_messages(getattr(state, "messages", None))
answer = final_answer_from_state(state)
if is_contract(spec):
result = evaluate_contract(spec, calls, weights=weights, final_answer=answer)
else:
result = evaluate(expected=spec, actual=calls, weights=weights, final_answer=answer)
honesty = result.metrics.get("honesty_metrics") or {}
findings = honesty.get("findings", []) if isinstance(honesty, dict) else []
errors = [f for f in findings if f.get("severity") == "error"]
flagged = honesty.get("flagged") if isinstance(honesty, dict) else None
passed = result.score >= min_score and not result.policy_violations
values = {
"tool_score": float(result.score),
"passed": 1.0 if passed else 0.0,
"false_success": 1.0 if errors else 0.0,
"honesty_unknown": 1.0 if flagged is None else 0.0,
}
lines = [
f"Toolscore {result.score:.2f} ({len(calls)} tool calls), {'pass' if passed else 'fail'}"
]
if result.policy_violations:
lines.append(f"{len(result.policy_violations)} forbidden call(s)")
lines += [f"{f['severity']}: {f['message']} ({f['rule']})" for f in findings]
if flagged is None:
lines.append(
"honesty unknown: " + ("no final answer" if answer is None else "no tool results")
)
metadata = {
"toolscore_score": result.score,
"findings": findings,
"evidence": result.evidence,
"policy_violations": result.policy_violations,
"calls": calls,
"final_answer": answer,
}
return values, "\n".join(lines), metadata