{
  "_comment": "Eval suite for skill output quality and trigger testing. See references/description-optimization.md for the full optimization loop.",
  "skill_name": "my-skill",
  "trigger_queries": {
    "_comment": "~20 queries split 60/40 into train and validation sets. Each has should_trigger: true/false.",
    "train": [
      { "query": "I have a CSV of monthly sales data in ~/data/q4_results.csv — can you find the top 3 months by revenue?", "should_trigger": true },
      { "query": "my boss wants a chart from this data file", "should_trigger": true },
      { "query": "can you analyze this spreadsheet and tell me which rows have missing values?", "should_trigger": true },
      { "query": "whats the quickest way to convert this json file to yaml", "should_trigger": false },
      { "query": "write a fibonacci function in python", "should_trigger": false },
      { "query": "can you write a python script that reads a csv and uploads each row to postgres", "should_trigger": false }
    ],
    "validation": [
      { "query": "hey can you clean up this csv and remove duplicates?", "should_trigger": true },
      { "query": "I need to update the formulas in my Excel budget spreadsheet", "should_trigger": false }
    ]
  },
  "output_evals": {
    "_comment": "Test cases for evaluating skill output quality. Store input files in evals/files/.",
    "evals": [
      {
        "id": 1,
        "prompt": "I have a CSV of monthly sales data in data/sales_2025.csv. Can you find the top 3 months by revenue and make a bar chart?",
        "expected_output": "A bar chart image showing the top 3 months by revenue, with labeled axes and values.",
        "files": ["evals/files/sales_2025.csv"],
        "assertions": [
          "The output includes a bar chart image file",
          "The chart shows exactly 3 months",
          "Both axes are labeled",
          "The chart title or caption mentions revenue"
        ]
      },
      {
        "id": 2,
        "prompt": "there's a csv in my downloads called customers.csv, some rows have missing emails — can you clean it up and tell me how many were missing?",
        "expected_output": "A cleaned CSV with missing emails handled, plus a count of how many were missing.",
        "files": ["evals/files/customers.csv"],
        "assertions": [
          "Output includes a cleaned CSV file",
          "Output states the count of rows with missing emails"
        ]
      }
    ]
  }
}