docs: publish target AI adoption strategy and synthetic measurement examples

scripts/ai-kpi.mjs

@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+/** SYNTHETIC reference only. No collectors, network access or production authorization.
+ * Input is a deliberately simplified, reconciled fixture, NOT a production event schema.
+ * Conflicts fail closed (no output); reconciliation/quarantine storage is not implemented.
+ */
+import { readFileSync } from 'node:fs';
+import { pathToFileURL } from 'node:url';
+export const LABEL = 'SYNTHETIC — illustrative, not Eldorado observations';
+export const CALCULATION_VERSION = 'synthetic-1.0.0';
+export const ratio = (n, d, scale = 100) => d === 0 ? null : n / d * scale;
+export function time(value) {
+  if (typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|\+00:00)$/.test(value)) throw new Error('UTC timestamp required');
+  const t = Date.parse(value);
+  if (!Number.isFinite(t) || new Date(t).toISOString().slice(0,19) !== value.slice(0,19)) throw new Error('Invalid timestamp');
+  return t;
+}
+const canonical = value => JSON.stringify(value, (_, v) => v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a],[b]) => a.localeCompare(b))) : v);
+const without = (row, keys) => Object.fromEntries(Object.entries(row).filter(([k]) => !keys.includes(k)));
+export function dedup(events) {
+  const kept = new Map();
+  for (const e of events) {
+    if (!e.producer || !e.event_id) throw new Error('Missing transport identity');
+    if (time(e.received_at) < time(e.occurred_at)) throw new Error('Receipt before occurrence');
+    const key = canonical([e.producer, e.event_id]), prior = kept.get(key);
+    if (prior && canonical(without(prior,['received_at'])) !== canonical(without(e,['received_at']))) throw new Error('Conflicting duplicate requires quarantine');
+    if (!prior || time(e.received_at) < time(prior.received_at)) kept.set(key,e);
+  }
+  return [...kept.values()];
+}
+function result(n, d, as_of, {unit = '%', scale = 100, unknown = 0, exclusions = {}, ...extra} = {}) {
+  return {label:LABEL, as_of, calculation_version:CALCULATION_VERSION, numerator:n, denominator:d, unit, value_or_null:ratio(n,d,scale), unknown_count:unknown, exclusions, coverage:{known_percent:ratio(d-unknown,d)}, warnings:unknown ? ['Unknown records remain visible; this is not a failure/non-use rate.'] : [], ...extra};
+}
+function taskCohort(events, start, end, as_of) {
+  const lo=time(start), hi=time(end), cutoff=time(as_of);
+  if (lo >= hi) throw new Error('Invalid half-open window');
+  const starts=new Map(), terminal=new Map();
+  for (const e of dedup(events)) {
+    if (time(e.occurred_at)>cutoff || time(e.received_at)>cutoff) continue;
+    if (e.event_type.startsWith('task.') && !e.task_id) throw new Error('Missing task ID');
+    if (e.event_type==='task.started') {
+      if (!Number.isFinite(e.horizon_hours) || e.horizon_hours<0) throw new Error('Missing/invalid observation horizon');
+      const prior=starts.get(e.task_id);
+      if (prior && (time(prior.occurred_at)!==time(e.occurred_at) || prior.horizon_hours!==e.horizon_hours)) throw new Error('Conflicting semantic task start');
+      starts.set(e.task_id,e);
+    } else if (['task.accepted','task.failed','task.abandoned','task.cancelled'].includes(e.event_type)) {
+      if (e.event_type==='task.accepted' && e.independent_evidence!==true) continue;
+      const prior=terminal.get(e.task_id);
+      if (prior && prior.event_type!==e.event_type) throw new Error('Conflicting task terminal state');
+      if (!prior || time(e.occurred_at)<time(prior.occurred_at)) terminal.set(e.task_id,e);
+    }
+  }
+  for (const [id,e] of terminal) if (starts.has(id) && time(e.occurred_at)<time(starts.get(id).occurred_at)) throw new Error('Terminal before start');
+  const eligible = new Set([...starts].filter(([,e]) => time(e.occurred_at)>=lo && time(e.occurred_at)<hi && time(e.occurred_at)+e.horizon_hours*3600000<=cutoff).map(([id])=>id));
+  return {starts, terminal, eligible};
+}
+// Snapshot rows are pre-reconciled at fixture as_of. Optional receipt/occurrence
+// timestamps support late-record tests; missing timestamps are NOT inferred events.
+function unique(rows, fields=['id']) {
+  const map=new Map();
+  for (const r of rows) {
+    if (fields.some(k=>typeof r[k]!=='string' || !r[k])) throw new Error('Missing semantic identity');
+    const key=canonical(fields.map(k=>r[k])), payload=without(r,['producer','event_id','received_at']);
+    if (map.has(key) && canonical(without(map.get(key),['producer','event_id','received_at']))!==canonical(payload)) throw new Error('Semantic conflict requires reconciliation');
+    const prior=map.get(key);
+    if (!prior || (r.received_at && prior.received_at && time(r.received_at)<time(prior.received_at))) map.set(key,r);
+  }
+  return [...map.values()];
+}
+const visible = (r, as_of) => (!r.received_at || time(r.received_at)<=time(as_of)) && (!r.occurred_at || time(r.occurred_at)<=time(as_of));
+const within = (t,start,end) => time(t)>=time(start) && time(t)<time(end);
+function horizon(r) {
+  if (!Number.isFinite(r.followup_hours) || r.followup_hours<0) throw new Error('Missing/invalid followup horizon');
+  return r.followup_hours*3600000;
+}
+export function association(rows) {
+  const map=new Map();
+  for (const r of rows) {
+    const tuple={skill_id:r.skill_id,revision:r.revision ?? null,version_status:!Object.hasOwn(r,'revision') ? 'missing' : r.revision===null ? 'unknown' : 'known',client:r.client,missing_reason:r.missing_reason ?? null};
+    if (!map.has(r.task_id)) map.set(r.task_id,new Map());
+    map.get(r.task_id).set(canonical(tuple),tuple);
+  }
+  return Object.fromEntries([...map].map(([id, tuples])=>[id,[...tuples.values()].sort((a,b)=>canonical(a).localeCompare(canonical(b)))]));
+}
+export function deployments(rows) {
+  const effects=unique(rows,['service_id','deployment_id']).filter(r=>r.environment==='production' && r.production_effect===true);
+  return {effects:effects.length,successful:effects.filter(r=>r.succeeded===true).length,change_fail_rate:ratio(effects.filter(r=>r.intervention===true).length,effects.length),rework_rate:ratio(effects.filter(r=>r.planned===false && Boolean(r.incident_id)).length,effects.length)};
+}
+export function matureDefects(changes,defects,as_of,start='0001-01-01T00:00:00Z',end=as_of) {
+  const eligible=unique(changes).filter(c=>visible(c,as_of) && within(c.accepted_at,start,end) && time(c.accepted_at)+horizon(c)<=time(as_of));
+  const affected=new Set();
+  for (const d of unique(defects)) {
+    const c=eligible.find(c=>c.id===d.change_id);
+    if (c && d.confirmed===true && visible(d,as_of) && time(d.detected_at)>=time(c.accepted_at) && time(d.detected_at)<=time(c.accepted_at)+horizon(c)) affected.add(c.id);
+  }
+  return [affected.size,eligible.length];
+}
+export function survey(rows,invited,as_of) {
+  if (!Number.isInteger(invited) || invited<0) throw new Error('Invalid invitation count');
+  const all=unique(rows), valid=all.filter(r=>Number.isInteger(r.score) && r.score>=1 && r.score<=5);
+  if (all.length>invited) throw new Error('Responses exceed invitations');
+  const suppressed=valid.length<5;
+  return result(suppressed?null:valid.reduce((n,r)=>n+r.score,0),valid.length,as_of,{unit:'points (1–5)',scale:1,value_or_null:suppressed?null:ratio(valid.reduce((n,r)=>n+r.score,0),valid.length,1),suppressed,exclusions:{invalid_answers:all.length-valid.length},coverage:{response_percent:ratio(valid.length,invited)},distribution:suppressed?null:Object.fromEntries([1,2,3,4,5].map(s=>[s,valid.filter(r=>r.score===s).length])),warnings:suppressed?['People cohort below five suppressed.']:[]});
+}
+export function calculate(f) {
+  if (f.synthetic!==true || f.schema_version!=='1.0') throw new Error('Explicit synthetic schema 1.0 fixture required');
+  const {start,end,as_of}=f, kpis={};
+  kpis['validated-task-success']=success(f.events,start,end,as_of);
+  const trials=unique(f.selection).filter(r=>visible(r,as_of));
+  const scored=trials.filter(r=>typeof r.relevant==='boolean' && typeof r.activated==='boolean');
+  const tp=scored.filter(r=>r.relevant && r.activated).length;
+  for (const [id,den] of [['selection-precision',scored.filter(r=>r.activated).length],['selection-recall',scored.filter(r=>r.relevant).length]]) {
+    kpis[id]=result(tp,den,as_of,{unknown:trials.length-scored.length,coverage:{scorable_percent:ratio(scored.length,trials.length)},exclusions:{unscorable:trials.length-scored.length}});
+  }
+  const effects=unique(f.deployments,['service_id','deployment_id']).filter(r=>visible(r,as_of) && r.environment==='production' && r.production_effect===true && within(r.deployed_at,start,end));
+  const mature=effects.filter(r=>time(r.deployed_at)+horizon(r)<=time(as_of));
+  kpis['change-fail-rate']=result(mature.filter(r=>r.intervention===true).length,mature.length,as_of,{unknown:mature.filter(r=>typeof r.intervention!=='boolean').length,exclusions:{immature:effects.length-mature.length}});
+  kpis['deployment-rework-rate']=result(effects.filter(r=>r.planned===false && Boolean(r.incident_id)).length,effects.length,as_of,{unknown:effects.filter(r=>typeof r.planned!=='boolean' || (r.planned===false && !Object.hasOwn(r,'incident_id'))).length});
+  const [affected,accepted]=matureDefects(f.changes,f.defects,as_of,start,end);
+  const matureChanges=unique(f.changes).filter(r=>visible(r,as_of) && within(r.accepted_at,start,end) && time(r.accepted_at)+horizon(r)<=time(as_of));
+  kpis['escaped-quality-rate']=result(affected,accepted,as_of,{unknown:matureChanges.filter(r=>r.linkage_complete!==true).length,exclusions:{outside_window_immature_or_late:f.changes.length-matureChanges.length},pending_triage_count:unique(f.defects).filter(r=>visible(r,as_of) && r.confirmed!==true).length});
+  const attempts=unique(f.attempts).filter(r=>visible(r,as_of));
+  kpis['policy-effects']=result(attempts.filter(r=>r.actual_effect===true).length,attempts.length,as_of,{unknown:attempts.filter(r=>typeof r.actual_effect!=='boolean').length,blocked_count:attempts.filter(r=>r.blocked===true).length});
+  const {eligible}=taskCohort(f.events,start,end,as_of);
+  const costs=unique(f.costs).filter(r=>visible(r,as_of) && (eligible.has(r.task_id) || r.task_id==='shared_pool'));
+  if (costs.some(r=>r.currency!==f.currency || !Number.isFinite(r.amount) || r.amount<0)) throw new Error('Invalid cost or mixed currency: approved conversion required');
+  const missing=f.expected_cost_buckets.filter(b=>!costs.some(r=>r.bucket===b));
+  kpis['cost-per-accepted-outcome']=result(costs.reduce((sum,r)=>sum+r.amount,0),kpis['validated-task-success'].numerator,as_of,{scale:1,unit:`${f.currency}/accepted outcome`,partial_estimate:missing.length>0,coverage:{missing_cost_buckets:missing},warnings:missing.length?['Partial estimate, not total cost; missing cost buckets.']:[]});
+  kpis['human-experience']=survey(f.survey.responses.filter(r=>visible(r,as_of)),f.survey.invited,as_of);
+  return {label:LABEL,as_of,calculation_version:CALCULATION_VERSION,window:{start,end},kpis,associations:association(f.associations),not_implemented:['supported-bundle-coverage','useful-adoption','change-lead-time','deployment-frequency','failed-deployment-recovery-time','maintenance-health','department-outcome'],limitations:['Synthetic simplified reconciled snapshots only; not the full production event envelope.','Snapshot rows without timestamps are assumed frozen at as_of, not missing-event evidence.','No causal attribution, uncertainty intervals, authorization or telemetry collection.','Survey implements primary item only; no cross-slice disclosure protection.','Defect severity strata and policy severity counts are not implemented.']};
+}
+if (process.argv[1] && import.meta.url===pathToFileURL(process.argv[1]).href) {
+  try {
+    if (process.argv.length>3) throw new Error('Usage: node scripts/ai-kpi.mjs [synthetic-fixture.json]');
+    const source=process.argv[2] ?? new URL('./fixtures/ai-kpi-synthetic.json',import.meta.url);
+    console.log(JSON.stringify(calculate(JSON.parse(readFileSync(source,'utf8'))),null,2));
+  } catch (error) { console.error(`Synthetic KPI calculation failed: ${error.message}`); process.exitCode=1; }
+}
+export function success(events,start,end,as_of) {
+  const {starts,terminal,eligible}=taskCohort(events,start,end,as_of);
+  const n=[...eligible].filter(id=>terminal.get(id)?.event_type==='task.accepted').length;
+  const unknown=[...eligible].filter(id=>!terminal.has(id)).length;
+  return result(n,eligible.size,as_of,{unknown, known_outcome_coverage:ratio(eligible.size-unknown,eligible.size), orphan_terminal_count:[...terminal.keys()].filter(id=>!starts.has(id)).length, excluded_start_count:starts.size-eligible.size, exclusions:{outside_window_or_immature:starts.size-eligible.size}});
+}