oleg-lukasonokandClaude Fable 5
6042272ccc
- Skill content moved from ~/.agents/skills/skill-creator, renamed to skill-manager - New deploy CLI action: installs any skill via absolute-path symlinks (or copies) into $HOME/.agents/skills, $HOME/.claude/skills and $HOME/.cline/skills - Taskfile + shell module wrappers (task deploy / cli:deploy) - SKILL.md: deploy docs, origin-repository/origin-path metadata Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
47 lines
2.4 KiB
JSON
47 lines
2.4 KiB
JSON
{
|
|
"_comment": "Eval suite for skill output quality and trigger testing. See references/description-optimization.md for the full optimization loop.",
|
|
"skill_name": "my-skill",
|
|
"trigger_queries": {
|
|
"_comment": "~20 queries split 60/40 into train and validation sets. Each has should_trigger: true/false.",
|
|
"train": [
|
|
{ "query": "I have a CSV of monthly sales data in ~/data/q4_results.csv — can you find the top 3 months by revenue?", "should_trigger": true },
|
|
{ "query": "my boss wants a chart from this data file", "should_trigger": true },
|
|
{ "query": "can you analyze this spreadsheet and tell me which rows have missing values?", "should_trigger": true },
|
|
{ "query": "whats the quickest way to convert this json file to yaml", "should_trigger": false },
|
|
{ "query": "write a fibonacci function in python", "should_trigger": false },
|
|
{ "query": "can you write a python script that reads a csv and uploads each row to postgres", "should_trigger": false }
|
|
],
|
|
"validation": [
|
|
{ "query": "hey can you clean up this csv and remove duplicates?", "should_trigger": true },
|
|
{ "query": "I need to update the formulas in my Excel budget spreadsheet", "should_trigger": false }
|
|
]
|
|
},
|
|
"output_evals": {
|
|
"_comment": "Test cases for evaluating skill output quality. Store input files in evals/files/.",
|
|
"evals": [
|
|
{
|
|
"id": 1,
|
|
"prompt": "I have a CSV of monthly sales data in data/sales_2025.csv. Can you find the top 3 months by revenue and make a bar chart?",
|
|
"expected_output": "A bar chart image showing the top 3 months by revenue, with labeled axes and values.",
|
|
"files": ["evals/files/sales_2025.csv"],
|
|
"assertions": [
|
|
"The output includes a bar chart image file",
|
|
"The chart shows exactly 3 months",
|
|
"Both axes are labeled",
|
|
"The chart title or caption mentions revenue"
|
|
]
|
|
},
|
|
{
|
|
"id": 2,
|
|
"prompt": "there's a csv in my downloads called customers.csv, some rows have missing emails — can you clean it up and tell me how many were missing?",
|
|
"expected_output": "A cleaned CSV with missing emails handled, plus a count of how many were missing.",
|
|
"files": ["evals/files/customers.csv"],
|
|
"assertions": [
|
|
"Output includes a cleaned CSV file",
|
|
"Output states the count of rows with missing emails"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|