documentation/scripts/ai-kpi.test.mjs

import test from 'node:test';
import assert from 'node:assert/strict';
import { existsSync, readFileSync } from 'node:fs';
const moduleUrl = new URL('./ai-kpi.mjs', import.meta.url);
import { spawnSync } from 'node:child_process';
import * as api from './ai-kpi.mjs';
const fixture = () => JSON.parse(readFileSync(new URL('./fixtures/ai-kpi-synthetic.json', import.meta.url)));
test('semantic change duplicates preserve the complete quality report including exclusions', () => {
  const original = fixture();
  const repeated = fixture();
  repeated.changes.push({...repeated.changes[0]});
  assert.deepEqual(api.calculate(repeated).kpis['escaped-quality-rate'], api.calculate(original).kpis['escaped-quality-rate']);
});
test('CLI report exposes implemented KPI contracts and explicit omissions', async () => {
  const m = await import(moduleUrl);
  assert.equal(typeof m.calculate, 'function');
  const r = m.calculate(fixture());
  assert.equal(Object.keys(r.kpis).length, 9);
  assert.equal(r.not_implemented.length, 7);
  assert.equal(r.kpis['cost-per-accepted-outcome'].value_or_null, 60);
  assert.equal(r.kpis['selection-precision'].value_or_null, 50);
  assert.equal(r.kpis['selection-recall'].value_or_null, 50);
  assert.equal(r.kpis['change-fail-rate'].value_or_null, 100);
  assert.equal(r.kpis['deployment-rework-rate'].value_or_null, 100);
  assert.equal(r.kpis['escaped-quality-rate'].value_or_null, 100);
  assert.equal(r.kpis['policy-effects'].value_or_null, 50);
  assert.equal(r.kpis['human-experience'].value_or_null, 4);
});
function event(id,task,kind,day=1,extra={}) {
  const timestamp=`2026-01-${String(day).padStart(2,'0')}T00:00:00Z`;
  return {producer:'synthetic',event_id:id,task_id:task,event_type:kind,occurred_at:timestamp,received_at:timestamp,...extra};
}
const run = (events,as_of=fixture().as_of) => api.success(events,fixture().start,fixture().end,as_of);
test('transport duplicates and redelivery use earliest receipt in either order',()=>{
  const f=fixture(), duplicate={...f.events[0],received_at:'2026-01-06T00:00:00Z'};
  assert.deepEqual(run([...f.events,f.events[0]]),run(f.events));
  assert.deepEqual(run([...f.events,duplicate]),run(f.events));
  assert.deepEqual(run([duplicate,...f.events]),run(f.events));
});
test('transport conflicts fail closed',()=>assert.throws(()=>run([...fixture().events,{...fixture().events[0],task_id:'other'}]),/Conflicting duplicate/));
test('retry and cross-feed semantic duplicate never add tasks',()=>{
  const f=fixture();
  assert.equal(run([...f.events,event('retry','accepted','attempt.started',2),event('retry-fail','accepted','attempt.failed',2),{...f.events[0],producer:'mirror',event_id:'mirror-start'}]).denominator,4);
});
test('immature task and half-open end excluded',()=>{
  const r=run([...fixture().events,event('boundary','boundary','task.started',5,{horizon_hours:0})]);
  assert.equal(r.excluded_start_count,2);
  assert.equal(r.denominator,4);
});
test('all implemented zero denominator values are null',()=>{
  const f=fixture();
  for (const k of ['events','selection','deployments','changes','defects','attempts','costs','associations']) f[k]=[];
  f.survey={invited:0,responses:[]};
  for (const r of Object.values(api.calculate(f).kpis)) assert.equal(r.value_or_null,null);
});
test('orphan terminal does not increase accepted count',()=>{
  const r=run([...fixture().events,event('orphan','absent','task.accepted',2,{independent_evidence:true})]);
  assert.equal(r.orphan_terminal_count,1); assert.equal(r.numerator,1);
});
test('late arrival restates only when received',()=>{
  const events=[...fixture().events,event('late','unknown','task.accepted',3,{independent_evidence:true,received_at:'2026-01-06T00:00:00Z'})];
  assert.equal(run(events).numerator,1);
  assert.equal(run(events,'2026-01-06T00:00:00Z').numerator,2);
});
test('unknown, absent and mixed versions retained without latest inference',()=>{
  const rows=['a','b','a',null].map(revision=>({task_id:'t',skill_id:'s',revision,client:'Cline'}));
  rows.push({task_id:'t',skill_id:'s',client:'Cline'});
  const r=api.association(rows).t;
  assert.equal(r.length,4);
  assert.equal(r.filter(x=>x.version_status==='known').length,2);
  assert.deepEqual(new Set(r.filter(x=>x.revision===null).map(x=>x.version_status)),new Set(['missing','unknown']));
});
test('invalid timestamps, UTC offsets, receipt order and terminal order rejected',()=>{
  for(const t of ['no','2026-02-30T00:00:00Z','2026-01-01T00:00:00+01:00']) assert.throws(()=>api.time(t));
  assert.throws(()=>run([event('s','t','task.started',2,{horizon_hours:24}),event('a','t','task.accepted',1,{independent_evidence:true})]),/Terminal before start/);
  assert.throws(()=>api.dedup([event('s','t','task.started',2,{received_at:'2026-01-01T00:00:00Z'})]),/Receipt before/);
});
test('missing horizon is not silently mature',()=>assert.throws(()=>run([event('s','t','task.started')]),/horizon/));
test('conflicting terminal and semantic start require reconciliation',()=>{
  assert.throws(()=>run([...fixture().events,event('conflict','accepted','task.failed',3)]),/Conflicting task terminal/);
  assert.throws(()=>run([...fixture().events,event('conflict','accepted','task.started',2,{horizon_hours:48})]),/semantic task start/);
});
test('no independent acceptance evidence means unknown, not accepted',()=>{
  const r=run([event('s','t','task.started',1,{horizon_hours:24}),event('a','t','task.accepted',2)]);
  assert.equal(r.numerator,0); assert.equal(r.unknown_count,1);
});
test('pre-start cancellation excluded; post-start cancellation remains denominator',()=>{
  const r=run([event('pre','never','task.cancelled',1,{before_start:true}),event('s','started','task.started',1,{horizon_hours:24}),event('post','started','task.cancelled',2,{before_start:false})]);
  assert.deepEqual([r.numerator,r.denominator,r.unknown_count],[0,1,0]);
});
test('deployment retries collapse; unknown rework not treated as planned',()=>{
  const row=fixture().deployments[0];
  assert.deepEqual(api.deployments([row,{...row,producer:'mirror',event_id:'2'}]),{effects:1,successful:1,change_fail_rate:100,rework_rate:100});
  assert.equal(api.deployments([{...row,planned:null}]).rework_rate,0);
  const f=fixture(); f.deployments[0].planned=null;
  const r=api.calculate(f).kpis['deployment-rework-rate'];
  assert.equal(r.denominator,1); assert.equal(r.unknown_count,1);
  assert.throws(()=>api.deployments([row,{...row,planned:true}]),/Semantic conflict/);
});
test('mature deployment denominator retains unsuccessful production effects',()=>{
  const f=fixture(); f.deployments[0].succeeded=false;
  assert.equal(api.calculate(f).kpis['change-fail-rate'].denominator,1);
  f.deployments[0].followup_hours=1000;
  assert.equal(api.calculate(f).kpis['change-fail-rate'].denominator,0);
});
test('mature defect followup is linked, bounded and deduplicated',()=>{
  const f=fixture();
  f.changes.push({id:'new',accepted_at:'2026-01-04T00:00:00Z',followup_hours:48});
  f.defects.push({...f.defects[0],id:'new-defect',change_id:'new'});
  assert.deepEqual(api.matureDefects(f.changes,[...f.defects,f.defects[0]],f.as_of),[1,1]);
  f.defects[0].detected_at='2026-01-04T00:00:00Z';
  assert.deepEqual(api.matureDefects(f.changes,f.defects,f.as_of),[0,1]);
});
test('cost includes failed attempts, flags partial buckets and rejects mixed currencies',()=>{
  const f=fixture(); let r=api.calculate(f).kpis['cost-per-accepted-outcome'];
  assert.equal(r.numerator,60); assert.equal(r.partial_estimate,true);
  f.costs.push({...f.costs[0]});
  assert.equal(api.calculate(f).kpis['cost-per-accepted-outcome'].numerator,60);
  f.costs.pop(); // Currency validation is separate from conflicting duplicate validation.
  f.costs[0].currency='USD'; assert.throws(()=>api.calculate(f),/mixed currency/);
});
test('blocked attempts are not prohibited effects; unknown audits remain denominator',()=>{
  const f=fixture(); f.attempts.push({id:'unknown'});
  const r=api.calculate(f).kpis['policy-effects'];
  assert.deepEqual([r.numerator,r.denominator,r.blocked_count,r.unknown_count],[1,3,1,1]);
});
test('survey suppresses small groups including sum and distribution',()=>{
  const f=fixture(), r=api.survey(f.survey.responses.slice(0,4),10,f.as_of);
  assert.equal(r.value_or_null,null); assert.equal(r.numerator,null); assert.equal(r.distribution,null);
});
test('required output fields accompany every implemented KPI',()=>{
  const required=['label','as_of','calculation_version','numerator','denominator','unit','value_or_null','unknown_count','exclusions','coverage','warnings'];
  for (const r of Object.values(api.calculate(fixture()).kpis)) for(const key of required) assert.ok(Object.hasOwn(r,key),key);
});
test('non-synthetic and unknown fixture schemas rejected',()=>{
  assert.throws(()=>api.calculate({...fixture(),synthetic:false}),/synthetic/);
  assert.throws(()=>api.calculate({...fixture(),schema_version:'future'}),/schema/);
});
test('CLI executes default and explicit fixture and fails visibly for missing input',()=>{
  for(const args of [[],[new URL('./fixtures/ai-kpi-synthetic.json',import.meta.url).pathname]]) {
    const r=spawnSync(process.execPath,[moduleUrl.pathname,...args],{encoding:'utf8'});
    assert.equal(r.status,0,r.stderr); assert.equal(JSON.parse(r.stdout).kpis['validated-task-success'].value_or_null,25);
  }
  const r=spawnSync(process.execPath,[moduleUrl.pathname,'/nonexistent/synthetic-fixture.json'],{encoding:'utf8'});
  assert.equal(r.status,1); assert.equal(r.stdout,''); assert.match(r.stderr,/failed/);
});
test('synthetic task calculator exists and retains failed, abandoned, unknown tasks', async () => {
  assert.ok(existsSync(moduleUrl), 'calculator implementation is required');
  const { success } = await import(moduleUrl);
  const f = JSON.parse(readFileSync(new URL('./fixtures/ai-kpi-synthetic.json', import.meta.url)));
  const r = success(f.events, f.start, f.end, f.as_of);
  assert.deepEqual([r.numerator, r.denominator, r.unknown_count, r.value_or_null, r.known_outcome_coverage], [1,4,1,25,75]);
});