docs: publish target AI adoption strategy and synthetic measurement examples

scripts/ai-kpi.test.mjs

@@ -0,0 +1,147 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { existsSync, readFileSync } from 'node:fs';
+const moduleUrl = new URL('./ai-kpi.mjs', import.meta.url);
+import { spawnSync } from 'node:child_process';
+import * as api from './ai-kpi.mjs';
+const fixture = () => JSON.parse(readFileSync(new URL('./fixtures/ai-kpi-synthetic.json', import.meta.url)));
+test('CLI report exposes implemented KPI contracts and explicit omissions', async () => {
+  const m = await import(moduleUrl);
+  assert.equal(typeof m.calculate, 'function');
+  const r = m.calculate(fixture());
+  assert.equal(Object.keys(r.kpis).length, 9);
+  assert.equal(r.not_implemented.length, 7);
+  assert.equal(r.kpis['cost-per-accepted-outcome'].value_or_null, 60);
+  assert.equal(r.kpis['selection-precision'].value_or_null, 50);
+  assert.equal(r.kpis['selection-recall'].value_or_null, 50);
+  assert.equal(r.kpis['change-fail-rate'].value_or_null, 100);
+  assert.equal(r.kpis['deployment-rework-rate'].value_or_null, 100);
+  assert.equal(r.kpis['escaped-quality-rate'].value_or_null, 100);
+  assert.equal(r.kpis['policy-effects'].value_or_null, 50);
+  assert.equal(r.kpis['human-experience'].value_or_null, 4);
+});
+function event(id,task,kind,day=1,extra={}) {
+  const timestamp=`2026-01-${String(day).padStart(2,'0')}T00:00:00Z`;
+  return {producer:'synthetic',event_id:id,task_id:task,event_type:kind,occurred_at:timestamp,received_at:timestamp,...extra};
+}
+const run = (events,as_of=fixture().as_of) => api.success(events,fixture().start,fixture().end,as_of);
+test('transport duplicates and redelivery use earliest receipt in either order',()=>{
+  const f=fixture(), duplicate={...f.events[0],received_at:'2026-01-06T00:00:00Z'};
+  assert.deepEqual(run([...f.events,f.events[0]]),run(f.events));
+  assert.deepEqual(run([...f.events,duplicate]),run(f.events));
+  assert.deepEqual(run([duplicate,...f.events]),run(f.events));
+});
+test('transport conflicts fail closed',()=>assert.throws(()=>run([...fixture().events,{...fixture().events[0],task_id:'other'}]),/Conflicting duplicate/));
+test('retry and cross-feed semantic duplicate never add tasks',()=>{
+  const f=fixture();
+  assert.equal(run([...f.events,event('retry','accepted','attempt.started',2),event('retry-fail','accepted','attempt.failed',2),{...f.events[0],producer:'mirror',event_id:'mirror-start'}]).denominator,4);
+});
+test('immature task and half-open end excluded',()=>{
+  const r=run([...fixture().events,event('boundary','boundary','task.started',5,{horizon_hours:0})]);
+  assert.equal(r.excluded_start_count,2);
+  assert.equal(r.denominator,4);
+});
+test('all implemented zero denominator values are null',()=>{
+  const f=fixture();
+  for (const k of ['events','selection','deployments','changes','defects','attempts','costs','associations']) f[k]=[];
+  f.survey={invited:0,responses:[]};
+  for (const r of Object.values(api.calculate(f).kpis)) assert.equal(r.value_or_null,null);
+});
+test('orphan terminal does not increase accepted count',()=>{
+  const r=run([...fixture().events,event('orphan','absent','task.accepted',2,{independent_evidence:true})]);
+  assert.equal(r.orphan_terminal_count,1); assert.equal(r.numerator,1);
+});
+test('late arrival restates only when received',()=>{
+  const events=[...fixture().events,event('late','unknown','task.accepted',3,{independent_evidence:true,received_at:'2026-01-06T00:00:00Z'})];
+  assert.equal(run(events).numerator,1);
+  assert.equal(run(events,'2026-01-06T00:00:00Z').numerator,2);
+});
+test('unknown, absent and mixed versions retained without latest inference',()=>{
+  const rows=['a','b','a',null].map(revision=>({task_id:'t',skill_id:'s',revision,client:'Cline'}));
+  rows.push({task_id:'t',skill_id:'s',client:'Cline'});
+  const r=api.association(rows).t;
+  assert.equal(r.length,4);
+  assert.equal(r.filter(x=>x.version_status==='known').length,2);
+  assert.deepEqual(new Set(r.filter(x=>x.revision===null).map(x=>x.version_status)),new Set(['missing','unknown']));
+});
+test('invalid timestamps, UTC offsets, receipt order and terminal order rejected',()=>{
+  for(const t of ['no','2026-02-30T00:00:00Z','2026-01-01T00:00:00+01:00']) assert.throws(()=>api.time(t));
+  assert.throws(()=>run([event('s','t','task.started',2,{horizon_hours:24}),event('a','t','task.accepted',1,{independent_evidence:true})]),/Terminal before start/);
+  assert.throws(()=>api.dedup([event('s','t','task.started',2,{received_at:'2026-01-01T00:00:00Z'})]),/Receipt before/);
+});
+test('missing horizon is not silently mature',()=>assert.throws(()=>run([event('s','t','task.started')]),/horizon/));
+test('conflicting terminal and semantic start require reconciliation',()=>{
+  assert.throws(()=>run([...fixture().events,event('conflict','accepted','task.failed',3)]),/Conflicting task terminal/);
+  assert.throws(()=>run([...fixture().events,event('conflict','accepted','task.started',2,{horizon_hours:48})]),/semantic task start/);
+});
+test('no independent acceptance evidence means unknown, not accepted',()=>{
+  const r=run([event('s','t','task.started',1,{horizon_hours:24}),event('a','t','task.accepted',2)]);
+  assert.equal(r.numerator,0); assert.equal(r.unknown_count,1);
+});
+test('pre-start cancellation excluded; post-start cancellation remains denominator',()=>{
+  const r=run([event('pre','never','task.cancelled',1,{before_start:true}),event('s','started','task.started',1,{horizon_hours:24}),event('post','started','task.cancelled',2,{before_start:false})]);
+  assert.deepEqual([r.numerator,r.denominator,r.unknown_count],[0,1,0]);
+});
+test('deployment retries collapse; unknown rework not treated as planned',()=>{
+  const row=fixture().deployments[0];
+  assert.deepEqual(api.deployments([row,{...row,producer:'mirror',event_id:'2'}]),{effects:1,successful:1,change_fail_rate:100,rework_rate:100});
+  assert.equal(api.deployments([{...row,planned:null}]).rework_rate,0);
+  const f=fixture(); f.deployments[0].planned=null;
+  const r=api.calculate(f).kpis['deployment-rework-rate'];
+  assert.equal(r.denominator,1); assert.equal(r.unknown_count,1);
+  assert.throws(()=>api.deployments([row,{...row,planned:true}]),/Semantic conflict/);
+});
+test('mature deployment denominator retains unsuccessful production effects',()=>{
+  const f=fixture(); f.deployments[0].succeeded=false;
+  assert.equal(api.calculate(f).kpis['change-fail-rate'].denominator,1);
+  f.deployments[0].followup_hours=1000;
+  assert.equal(api.calculate(f).kpis['change-fail-rate'].denominator,0);
+});
+test('mature defect followup is linked, bounded and deduplicated',()=>{
+  const f=fixture();
+  f.changes.push({id:'new',accepted_at:'2026-01-04T00:00:00Z',followup_hours:48});
+  f.defects.push({...f.defects[0],id:'new-defect',change_id:'new'});
+  assert.deepEqual(api.matureDefects(f.changes,[...f.defects,f.defects[0]],f.as_of),[1,1]);
+  f.defects[0].detected_at='2026-01-04T00:00:00Z';
+  assert.deepEqual(api.matureDefects(f.changes,f.defects,f.as_of),[0,1]);
+});
+test('cost includes failed attempts, flags partial buckets and rejects mixed currencies',()=>{
+  const f=fixture(); let r=api.calculate(f).kpis['cost-per-accepted-outcome'];
+  assert.equal(r.numerator,60); assert.equal(r.partial_estimate,true);
+  f.costs.push({...f.costs[0]});
+  assert.equal(api.calculate(f).kpis['cost-per-accepted-outcome'].numerator,60);
+  f.costs.pop(); // Currency validation is separate from conflicting duplicate validation.
+  f.costs[0].currency='USD'; assert.throws(()=>api.calculate(f),/mixed currency/);
+});
+test('blocked attempts are not prohibited effects; unknown audits remain denominator',()=>{
+  const f=fixture(); f.attempts.push({id:'unknown'});
+  const r=api.calculate(f).kpis['policy-effects'];
+  assert.deepEqual([r.numerator,r.denominator,r.blocked_count,r.unknown_count],[1,3,1,1]);
+});
+test('survey suppresses small groups including sum and distribution',()=>{
+  const f=fixture(), r=api.survey(f.survey.responses.slice(0,4),10,f.as_of);
+  assert.equal(r.value_or_null,null); assert.equal(r.numerator,null); assert.equal(r.distribution,null);
+});
+test('required output fields accompany every implemented KPI',()=>{
+  const required=['label','as_of','calculation_version','numerator','denominator','unit','value_or_null','unknown_count','exclusions','coverage','warnings'];
+  for (const r of Object.values(api.calculate(fixture()).kpis)) for(const key of required) assert.ok(Object.hasOwn(r,key),key);
+});
+test('non-synthetic and unknown fixture schemas rejected',()=>{
+  assert.throws(()=>api.calculate({...fixture(),synthetic:false}),/synthetic/);
+  assert.throws(()=>api.calculate({...fixture(),schema_version:'future'}),/schema/);
+});
+test('CLI executes default and explicit fixture and fails visibly for missing input',()=>{
+  for(const args of [[],[new URL('./fixtures/ai-kpi-synthetic.json',import.meta.url).pathname]]) {
+    const r=spawnSync(process.execPath,[moduleUrl.pathname,...args],{encoding:'utf8'});
+    assert.equal(r.status,0,r.stderr); assert.equal(JSON.parse(r.stdout).kpis['validated-task-success'].value_or_null,25);
+  }
+  const r=spawnSync(process.execPath,[moduleUrl.pathname,'/nonexistent/synthetic-fixture.json'],{encoding:'utf8'});
+  assert.equal(r.status,1); assert.equal(r.stdout,''); assert.match(r.stderr,/failed/);
+});
+test('synthetic task calculator exists and retains failed, abandoned, unknown tasks', async () => {
+  assert.ok(existsSync(moduleUrl), 'calculator implementation is required');
+  const { success } = await import(moduleUrl);
+  const f = JSON.parse(readFileSync(new URL('./fixtures/ai-kpi-synthetic.json', import.meta.url)));
+  const r = success(f.events, f.start, f.end, f.as_of);
+  assert.deepEqual([r.numerator, r.denominator, r.unknown_count, r.value_or_null, r.known_outcome_coverage], [1,4,1,25,75]);
+});