Content hash: 2929f00177a186c17aa32c3a52b61c1844cdb4c0ac7b314e90e69196f4ede802
#!/usr/bin/env python3
"""Validate LLM tool schemas against best practices before shipping them.
Checks the concrete rules from the tool-schema-design skill: naming, enums,
required-field minimalism, per-parameter descriptions, and overlap detection.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
@dataclass
class ToolSchema:
name: str
description: str
parameters: dict[str, Any] # JSON Schema object
required: list[str] = field(default_factory=list)
@dataclass
class Finding:
tool: str
severity: str # "error" | "warn"
message: str
def validate_tool(tool: ToolSchema) -> list[Finding]:
findings: list[Finding] = []
props = tool.parameters.get("properties", {})
required = set(tool.required)
# 1. Description is the primary signal - must exist and be specific
if not tool.description or len(tool.description) < 20:
findings.append(Finding(
tool.name, "error",
"Description too short (<20 chars). It's the model's ONLY signal.",
))
# 2. Per-parameter descriptions
for pname, pdef in props.items():
if not pdef.get("description"):
findings.append(Finding(
tool.name, "warn", f"Parameter '{pname}' missing description.",
))
# 3. Enum for finite-value params (looks for string params w/o enum)
for pname, pdef in props.items():
if pdef.get("type") == "string" and "enum" not in pdef:
# Heuristic: params named like a mode/status/priority likely finite
if any(k in pname.lower() for k in ("mode", "status", "priority", "type", "level")):
findings.append(Finding(
tool.name, "warn",
f"'{pname}' looks finite-valued but has no 'enum' constraint.",
))
# 4. Required minimalism - every required field is a failure point
if len(required) > 3:
findings.append(Finding(
tool.name, "warn",
f"{len(required)} required params. Can any be deferred/defaulted?",
))
# 5. Unit/format hints for numeric/date params
for pname, pdef in props.items():
t = pdef.get("type")
if t in ("number", "integer") and not any(
k in pdef.get("description", "").lower() for k in ("ms", "sec", "bytes", "percent", "%", "unit")
):
findings.append(Finding(
tool.name, "warn",
f"Numeric param '{pname}' lacks a unit hint in its description.",
))
return findings
def detect_overlap(tools: list[ToolSchema]) -> list[Finding]:
"""Flag pairs of tools whose names/descriptions are too similar."""
findings: list[Finding] = []
for i in range(len(tools)):
for j in range(i + 1, len(tools)):
a, b = tools[i], tools[j]
shared = set(a.description.lower().split()) & set(b.description.lower().split())
name_similar = a.name.lower() in b.name.lower() or b.name.lower() in a.name.lower()
if name_similar and len(shared) > 4:
findings.append(Finding(
f"{a.name} / {b.name}", "warn",
"Overlapping tools - model will confuse them. Add boundary notes to descriptions.",
))
return findings
# ── Demo ───────────────────────────────────────────────────────────────────
GOOD_TOOL = ToolSchema(
name="get_calendar_events",
description="Retrieve calendar events for a date or range. Returns start/end times and titles.",
parameters={
"type": "object",
"properties": {
"start_date": {"type": "string", "description": "ISO 8601 date (YYYY-MM-DD)"},
"end_date": {"type": "string", "description": "ISO 8601 date (YYYY-MM-DD), optional"},
},
},
required=["start_date"],
)
BAD_TOOL = ToolSchema(
name="getData",
description="Gets data.",
parameters={
"type": "object",
"properties": {
"mode": {"type": "string"}, # no enum, no description
"uid": {"type": "integer"}, # no description, no unit
"x": {"type": "string"}, # no description
"y": {"type": "string"}, # no description
"z": {"type": "string"}, # no description
},
},
required=["mode", "uid", "x", "y", "z"], # 5 required = 5 failure points
)
def main() -> None:
print("=== Validating: GOOD tool ===")
for f in validate_tool(GOOD_TOOL):
print(f" [{f.severity}] {f.message}")
print("\n=== Validating: BAD tool ===")
for f in validate_tool(BAD_TOOL):
print(f" [{f.severity}] {f.message}")
print("\n=== Overlap detection ===")
overlapping = [
ToolSchema(
name="search_products",
description="Search products by keyword and return matching product listings with prices.",
parameters={"type": "object", "properties": {}},
),
ToolSchema(
name="search_products_by_price",
description="Search products by keyword and return matching product listings sorted by price.",
parameters={"type": "object", "properties": {}},
),
]
for f in detect_overlap(overlapping):
print(f" [{f.severity}] {f.message}")
if __name__ == "__main__":
main()