import test from 'node:test'; import assert from 'node:assert/strict'; import { existsSync, readFileSync } from 'node:fs'; const moduleUrl = new URL('./ai-kpi.mjs', import.meta.url); import { spawnSync } from 'node:child_process'; import * as api from './ai-kpi.mjs'; const fixture = () => JSON.parse(readFileSync(new URL('./fixtures/ai-kpi-synthetic.json', import.meta.url))); test('semantic change duplicates preserve the complete quality report including exclusions', () => { const original = fixture(); const repeated = fixture(); repeated.changes.push({...repeated.changes[0]}); assert.deepEqual(api.calculate(repeated).kpis['escaped-quality-rate'], api.calculate(original).kpis['escaped-quality-rate']); }); test('CLI report exposes implemented KPI contracts and explicit omissions', async () => { const m = await import(moduleUrl); assert.equal(typeof m.calculate, 'function'); const r = m.calculate(fixture()); assert.equal(Object.keys(r.kpis).length, 9); assert.equal(r.not_implemented.length, 7); assert.equal(r.kpis['cost-per-accepted-outcome'].value_or_null, 60); assert.equal(r.kpis['selection-precision'].value_or_null, 50); assert.equal(r.kpis['selection-recall'].value_or_null, 50); assert.equal(r.kpis['change-fail-rate'].value_or_null, 100); assert.equal(r.kpis['deployment-rework-rate'].value_or_null, 100); assert.equal(r.kpis['escaped-quality-rate'].value_or_null, 100); assert.equal(r.kpis['policy-effects'].value_or_null, 50); assert.equal(r.kpis['human-experience'].value_or_null, 4); }); function event(id,task,kind,day=1,extra={}) { const timestamp=`2026-01-${String(day).padStart(2,'0')}T00:00:00Z`; return {producer:'synthetic',event_id:id,task_id:task,event_type:kind,occurred_at:timestamp,received_at:timestamp,...extra}; } const run = (events,as_of=fixture().as_of) => api.success(events,fixture().start,fixture().end,as_of); test('transport duplicates and redelivery use earliest receipt in either order',()=>{ const f=fixture(), duplicate={...f.events[0],received_at:'2026-01-06T00:00:00Z'}; assert.deepEqual(run([...f.events,f.events[0]]),run(f.events)); assert.deepEqual(run([...f.events,duplicate]),run(f.events)); assert.deepEqual(run([duplicate,...f.events]),run(f.events)); }); test('transport conflicts fail closed',()=>assert.throws(()=>run([...fixture().events,{...fixture().events[0],task_id:'other'}]),/Conflicting duplicate/)); test('retry and cross-feed semantic duplicate never add tasks',()=>{ const f=fixture(); assert.equal(run([...f.events,event('retry','accepted','attempt.started',2),event('retry-fail','accepted','attempt.failed',2),{...f.events[0],producer:'mirror',event_id:'mirror-start'}]).denominator,4); }); test('immature task and half-open end excluded',()=>{ const r=run([...fixture().events,event('boundary','boundary','task.started',5,{horizon_hours:0})]); assert.equal(r.excluded_start_count,2); assert.equal(r.denominator,4); }); test('all implemented zero denominator values are null',()=>{ const f=fixture(); for (const k of ['events','selection','deployments','changes','defects','attempts','costs','associations']) f[k]=[]; f.survey={invited:0,responses:[]}; for (const r of Object.values(api.calculate(f).kpis)) assert.equal(r.value_or_null,null); }); test('orphan terminal does not increase accepted count',()=>{ const r=run([...fixture().events,event('orphan','absent','task.accepted',2,{independent_evidence:true})]); assert.equal(r.orphan_terminal_count,1); assert.equal(r.numerator,1); }); test('late arrival restates only when received',()=>{ const events=[...fixture().events,event('late','unknown','task.accepted',3,{independent_evidence:true,received_at:'2026-01-06T00:00:00Z'})]; assert.equal(run(events).numerator,1); assert.equal(run(events,'2026-01-06T00:00:00Z').numerator,2); }); test('unknown, absent and mixed versions retained without latest inference',()=>{ const rows=['a','b','a',null].map(revision=>({task_id:'t',skill_id:'s',revision,client:'Cline'})); rows.push({task_id:'t',skill_id:'s',client:'Cline'}); const r=api.association(rows).t; assert.equal(r.length,4); assert.equal(r.filter(x=>x.version_status==='known').length,2); assert.deepEqual(new Set(r.filter(x=>x.revision===null).map(x=>x.version_status)),new Set(['missing','unknown'])); }); test('invalid timestamps, UTC offsets, receipt order and terminal order rejected',()=>{ for(const t of ['no','2026-02-30T00:00:00Z','2026-01-01T00:00:00+01:00']) assert.throws(()=>api.time(t)); assert.throws(()=>run([event('s','t','task.started',2,{horizon_hours:24}),event('a','t','task.accepted',1,{independent_evidence:true})]),/Terminal before start/); assert.throws(()=>api.dedup([event('s','t','task.started',2,{received_at:'2026-01-01T00:00:00Z'})]),/Receipt before/); }); test('missing horizon is not silently mature',()=>assert.throws(()=>run([event('s','t','task.started')]),/horizon/)); test('conflicting terminal and semantic start require reconciliation',()=>{ assert.throws(()=>run([...fixture().events,event('conflict','accepted','task.failed',3)]),/Conflicting task terminal/); assert.throws(()=>run([...fixture().events,event('conflict','accepted','task.started',2,{horizon_hours:48})]),/semantic task start/); }); test('no independent acceptance evidence means unknown, not accepted',()=>{ const r=run([event('s','t','task.started',1,{horizon_hours:24}),event('a','t','task.accepted',2)]); assert.equal(r.numerator,0); assert.equal(r.unknown_count,1); }); test('pre-start cancellation excluded; post-start cancellation remains denominator',()=>{ const r=run([event('pre','never','task.cancelled',1,{before_start:true}),event('s','started','task.started',1,{horizon_hours:24}),event('post','started','task.cancelled',2,{before_start:false})]); assert.deepEqual([r.numerator,r.denominator,r.unknown_count],[0,1,0]); }); test('deployment retries collapse; unknown rework not treated as planned',()=>{ const row=fixture().deployments[0]; assert.deepEqual(api.deployments([row,{...row,producer:'mirror',event_id:'2'}]),{effects:1,successful:1,change_fail_rate:100,rework_rate:100}); assert.equal(api.deployments([{...row,planned:null}]).rework_rate,0); const f=fixture(); f.deployments[0].planned=null; const r=api.calculate(f).kpis['deployment-rework-rate']; assert.equal(r.denominator,1); assert.equal(r.unknown_count,1); assert.throws(()=>api.deployments([row,{...row,planned:true}]),/Semantic conflict/); }); test('mature deployment denominator retains unsuccessful production effects',()=>{ const f=fixture(); f.deployments[0].succeeded=false; assert.equal(api.calculate(f).kpis['change-fail-rate'].denominator,1); f.deployments[0].followup_hours=1000; assert.equal(api.calculate(f).kpis['change-fail-rate'].denominator,0); }); test('mature defect followup is linked, bounded and deduplicated',()=>{ const f=fixture(); f.changes.push({id:'new',accepted_at:'2026-01-04T00:00:00Z',followup_hours:48}); f.defects.push({...f.defects[0],id:'new-defect',change_id:'new'}); assert.deepEqual(api.matureDefects(f.changes,[...f.defects,f.defects[0]],f.as_of),[1,1]); f.defects[0].detected_at='2026-01-04T00:00:00Z'; assert.deepEqual(api.matureDefects(f.changes,f.defects,f.as_of),[0,1]); }); test('cost includes failed attempts, flags partial buckets and rejects mixed currencies',()=>{ const f=fixture(); let r=api.calculate(f).kpis['cost-per-accepted-outcome']; assert.equal(r.numerator,60); assert.equal(r.partial_estimate,true); f.costs.push({...f.costs[0]}); assert.equal(api.calculate(f).kpis['cost-per-accepted-outcome'].numerator,60); f.costs.pop(); // Currency validation is separate from conflicting duplicate validation. f.costs[0].currency='USD'; assert.throws(()=>api.calculate(f),/mixed currency/); }); test('blocked attempts are not prohibited effects; unknown audits remain denominator',()=>{ const f=fixture(); f.attempts.push({id:'unknown'}); const r=api.calculate(f).kpis['policy-effects']; assert.deepEqual([r.numerator,r.denominator,r.blocked_count,r.unknown_count],[1,3,1,1]); }); test('survey suppresses small groups including sum and distribution',()=>{ const f=fixture(), r=api.survey(f.survey.responses.slice(0,4),10,f.as_of); assert.equal(r.value_or_null,null); assert.equal(r.numerator,null); assert.equal(r.distribution,null); }); test('required output fields accompany every implemented KPI',()=>{ const required=['label','as_of','calculation_version','numerator','denominator','unit','value_or_null','unknown_count','exclusions','coverage','warnings']; for (const r of Object.values(api.calculate(fixture()).kpis)) for(const key of required) assert.ok(Object.hasOwn(r,key),key); }); test('non-synthetic and unknown fixture schemas rejected',()=>{ assert.throws(()=>api.calculate({...fixture(),synthetic:false}),/synthetic/); assert.throws(()=>api.calculate({...fixture(),schema_version:'future'}),/schema/); }); test('CLI executes default and explicit fixture and fails visibly for missing input',()=>{ for(const args of [[],[new URL('./fixtures/ai-kpi-synthetic.json',import.meta.url).pathname]]) { const r=spawnSync(process.execPath,[moduleUrl.pathname,...args],{encoding:'utf8'}); assert.equal(r.status,0,r.stderr); assert.equal(JSON.parse(r.stdout).kpis['validated-task-success'].value_or_null,25); } const r=spawnSync(process.execPath,[moduleUrl.pathname,'/nonexistent/synthetic-fixture.json'],{encoding:'utf8'}); assert.equal(r.status,1); assert.equal(r.stdout,''); assert.match(r.stderr,/failed/); }); test('synthetic task calculator exists and retains failed, abandoned, unknown tasks', async () => { assert.ok(existsSync(moduleUrl), 'calculator implementation is required'); const { success } = await import(moduleUrl); const f = JSON.parse(readFileSync(new URL('./fixtures/ai-kpi-synthetic.json', import.meta.url))); const r = success(f.events, f.start, f.end, f.as_of); assert.deepEqual([r.numerator, r.denominator, r.unknown_count, r.value_or_null, r.known_outcome_coverage], [1,4,1,25,75]); });