documentation/scripts/skills-measurement.test.mjs

import test from 'node:test';
import assert from 'node:assert/strict';
import {readFileSync} from 'node:fs';
import {calculate} from './ai-kpi.mjs';
const read = path => readFileSync(path, 'utf8');

test('Skills Measurement is a visible optional registry, separate from workflow application', () => {
  const page = read('docs/strategy/measurement/index.mdx');
  assert.match(page, /title: Skills Measurement\nsidebar_label: Measurement/);
  for (const heading of ['Establish the baseline', 'Measure the pilot', 'Extend collection as adoption grows', 'Keep the collection running']) assert(!page.includes(heading), heading);
  assert(page.includes('Registry inclusion does not require a metric to be used, collected or baselined'));
  assert(page.includes('../baseline.mdx'));
  assert(page.includes('<summary>Implementation status—not live telemetry</summary>'));
  const visible = page.replace(/<details>[\s\S]*?<\/details>/g, '');
  assert.equal([...visible.matchAll(/^\d+\. \[/gm)].length,16,'All contract links remain visible');
  assert(page.includes('./collection-and-safeguards.mdx'));
  assert(!page.includes('Is AI-assisted work worth continuing?'));
  for (const word of ['candidate', 'not installed', 'applicability', 'calculation', 'inputs', 'limitations', 'implementation status']) assert(page.includes(word), word);
});

test('all sixteen metric definitions remain in the reference with explicit implementation limits', () => {
  const manifest = JSON.parse(read('research/ai-adoption/measurement-migration.json'));
  const page = read('docs/strategy/measurement/index.mdx') + read('docs/strategy/measurement/collection-and-safeguards.mdx');
  const model = JSON.parse(read('research/ai-adoption/strategy-model.json'));
  for (const k of model.kpis) {
    const section = read(`docs/strategy/${manifest.metrics.find(m => m.title === k.name).docId}.mdx`);
    assert(section, k.name);
    for (const field of ['Numerator','Denominator','Expression','Aggregation','Unit','Eligible population','Exclusions','Events and source','Owner / cadence','Missing evidence','Gaming and limits']) assert(section.includes(`| ${field} |`));
  }
  for (const text of ['9 prototype calculations', '7 unimplemented metrics', 'no network collectors', '30 days', '12 months', 'below five people', 'Missing telemetry is unknown']) assert(page.includes(text), text);
});

test('two groups exhaustively classify the sixteen canonical contracts without duplication', () => {
  const page = read('docs/strategy/measurement/index.mdx');
  const manifest = JSON.parse(read('research/ai-adoption/measurement-migration.json'));
  assert.deepEqual([...page.matchAll(/^### (.+)$/gm)].map(m => m[1]), ['Workflow effectiveness', 'Skills effectiveness & health']);
  const rows = [...page.matchAll(/^\d+\. \[([^\]]+)\]\(\.\/([^/]+)\.mdx\) — \*\*(Prototype calculation|Unimplemented metric)\.\*\*/gm)];
  assert.equal(rows.length, 16);
  assert.equal(new Set(rows.map(m => m[2])).size, 16);
  assert.deepEqual(rows.map(m => m[1]).sort(), manifest.metrics.map(m => m.title).sort());
  assert.equal(rows.filter(m => m[3] === 'Prototype calculation').length, 9);
  assert.equal(rows.filter(m => m[3] === 'Unimplemented metric').length, 7);
  const [workflow, skills] = page.split('### Workflow effectiveness')[1].split('### Skills effectiveness & health');
  assert.equal([...workflow.matchAll(/^\d+\. /gm)].length, 11);
  assert.equal([...skills.matchAll(/^\d+\. /gm)].length, 5);
  for (const slug of ['validated-task-success', 'confirmed-policy-effects', 'cost-per-accepted-outcome', 'workflow-specific-business-acceptance']) assert(workflow.includes(`./${slug}.mdx`));
  for (const slug of ['skill-selection-precision', 'skill-selection-recall', 'supported-bundle-coverage', 'approved-workflow-adoption', 'skills-within-substantive-review-window']) assert(skills.includes(`./${slug}.mdx`));
  for (const marker of ['N/A, not zero', 'More adoption does not prove improved work', 'Before/after trends alone are not causal evidence', 'Sandbox/evaluation cases remain excluded']) assert(page.includes(marker), marker);
});

test('workflow task-success contract retains evaluation exclusions and explains conditional skill slices', () => {
  const page = read('docs/strategy/measurement/validated-task-success.mdx');
  assert.equal((page.match(/\| Expression \|/g) || []).length, 1);
  for (const marker of ['sandbox/evaluation tasks', 'Overlapping skill slices must not be summed', 'remains excluded by this contract', 'does not introduce an implemented skill-evaluation metric', 'N/A, not a zero workflow success rate']) assert(page.includes(marker), marker);
  for (const name of ['baseline', 'skills-adoption', 'skills-maintenance', 'stage-01-skills-adoption']) {
    const text = read(`docs/strategy/${name}.mdx`);
    if (name !== 'stage-01-skills-adoption') for (const group of ['Workflow effectiveness', 'Skills effectiveness & health']) assert(text.includes(group), name);
  }
});

test('published pilot arithmetic matches the executable fixture, including partial cost', () => {
  const page = read('docs/strategy/skills-adoption.mdx');
  const fixture = JSON.parse(read('scripts/fixtures/ai-kpi-synthetic.json'));
  const report = calculate(fixture);
  const success = report.kpis['validated-task-success'];
  assert(page.includes(`${success.numerator} / ${success.denominator} × 100 = ${success.value_or_null}%`));
  assert(page.includes(`${success.known_outcome_coverage}%`));
  const cost = report.kpis['cost-per-accepted-outcome'];
  assert.equal(cost.partial_estimate, true);
  assert(page.includes(`EUR ${cost.numerator} / ${cost.denominator} = EUR ${cost.value_or_null}`));
  assert(page.includes('partial estimate'));
  const experience = report.kpis['human-experience'];
  assert(page.includes(`${experience.numerator} / ${experience.denominator} = ${experience.value_or_null}`));
  assert(page.includes(`${experience.denominator} / ${fixture.survey.invited} × 100 = ${experience.coverage.response_percent}%`));
});