1168 lines
62 KiB
JSON
1168 lines
62 KiB
JSON
{
|
||
"schema_version": "1.0.0",
|
||
"status": "proposed-target-not-approved-or-deployed",
|
||
"scope": "Strategy documentation and synthetic measurement reference only; no runtime changes authorized.",
|
||
"organization": {
|
||
"Product": [
|
||
"Product Management",
|
||
"Product Design"
|
||
],
|
||
"Technology": [
|
||
"Product Engineering",
|
||
"Platform & IT",
|
||
"Architecture & Security",
|
||
"Quality Engineering",
|
||
"Data & AI"
|
||
],
|
||
"Operations": [
|
||
"Operations Squad"
|
||
],
|
||
"Marketing": [
|
||
"Marketing Squad"
|
||
],
|
||
"Finance & Accounting": [
|
||
"Finance & Accounting Squad"
|
||
],
|
||
"HR & Administration": [
|
||
"People & Administration Squad"
|
||
]
|
||
},
|
||
"pages": [
|
||
{
|
||
"id": "index",
|
||
"title": "AI adoption strategy",
|
||
"purpose": "Govern skills first; outcomes before activity; explicit target status."
|
||
},
|
||
{
|
||
"id": "target-operating-model",
|
||
"title": "Target operating model",
|
||
"purpose": "One logical coordinator; local client choice; independent authority; all departments."
|
||
},
|
||
{
|
||
"id": "skills-governance",
|
||
"title": "Skills governance",
|
||
"purpose": "Contract, lifecycle, decision rights, adapters, locking, exceptions."
|
||
},
|
||
{
|
||
"id": "skills-quality",
|
||
"title": "Skills quality",
|
||
"purpose": "Risk tiers, independent evaluation, repeated trials, adversarial conformance."
|
||
},
|
||
{
|
||
"id": "skills-adoption",
|
||
"title": "Skills adoption",
|
||
"purpose": "Workflow-led coaching, client choice, feedback, manual path."
|
||
},
|
||
{
|
||
"id": "measurement",
|
||
"title": "Measurement",
|
||
"purpose": "Full KPI contracts, minimal events, illustrative calculations, privacy and limits."
|
||
},
|
||
{
|
||
"id": "skills-maintenance",
|
||
"title": "Skills maintenance",
|
||
"purpose": "Review, drift, deprecation, rollback and incident handling."
|
||
},
|
||
{
|
||
"id": "roadmap",
|
||
"title": "Roadmap",
|
||
"purpose": "Evidence-gated phases, no promised dates or budgets."
|
||
}
|
||
],
|
||
"diagram": {
|
||
"logical_id": "ai-adoption-target",
|
||
"renderer": "existing Archify",
|
||
"editable_source": "diagrams/ai-adoption-target.architecture.json",
|
||
"public_json_download": false,
|
||
"coordinators": "one logical function, recoverable/replicated implementation permitted",
|
||
"forbidden_edges": [
|
||
"local client -> production authority",
|
||
"coordinator -> release authorization"
|
||
],
|
||
"required_boundaries": [
|
||
"human identity/approval",
|
||
"untrusted local work",
|
||
"registry supply chain",
|
||
"independent CI/release authority",
|
||
"model-provider data egress",
|
||
"restricted audit"
|
||
]
|
||
},
|
||
"skill_contract": {
|
||
"skill_id": "Stable globally unique eldorado-<domain>-<purpose>; never reused for a different purpose",
|
||
"name": "Human-readable name",
|
||
"purpose": "Problem, expected outcome and intended users",
|
||
"triggers": "Positive and negative invocation conditions",
|
||
"owner": "Existing department/team plus assigned accountable person before approval",
|
||
"reviewer": "Independent person and existing team; cannot equal release author",
|
||
"backup": "Named coverage within existing team before production support",
|
||
"risk_tier": "R1|R2|R3; maximum implied by data, tool and effect scope",
|
||
"data_classification": "Allowed input/output/provider boundary and forbidden data",
|
||
"authority_envelope": "Allowed resources/operations and forbidden effects; enforced externally",
|
||
"supported_clients": "Array of client, host, exact tested builds, OS, model configuration, adapter digest and qualification evidence",
|
||
"dependencies": "All skill/resource/script/tool/MCP/plugin dependencies pinned by immutable digest plus license and source",
|
||
"revision": "Full immutable source commit and content/artifact digest; friendly semver optional",
|
||
"tests_evals": "Suite/corpus/rubric/runner/judge revisions, client tuple, seeds or trial IDs, results and independent evidence refs",
|
||
"approval": "Approver identities, exact digest and policy revision, approved scope, time and expiry/invalidation condition",
|
||
"provenance_license": "Origin repository/author, upstream revision, acquisition/review history, rights/license for content and bundled resources",
|
||
"maintenance_date": "Last substantive review timestamp plus next review due and trigger conditions",
|
||
"deprecation": "State, reason, replacement, migration instructions, consumers and retirement deadline if applicable",
|
||
"rollback": "Approved known-good bundle+adapter+dependency digests, revocation/reload procedure and tested evidence",
|
||
"procedure": "Prerequisites, bounded steps, expected artifacts, verification, failure handling, escalation"
|
||
},
|
||
"lifecycle": {
|
||
"states": [
|
||
"proposed",
|
||
"experimental",
|
||
"approved",
|
||
"deprecated",
|
||
"retired",
|
||
"quarantined"
|
||
],
|
||
"transitions": [
|
||
{
|
||
"from": "proposed",
|
||
"to": "experimental",
|
||
"gate": "Owner, purpose, risk/data scope, disposable environment and preregistered test plan"
|
||
},
|
||
{
|
||
"from": "experimental",
|
||
"to": "approved",
|
||
"gate": "Independent review, all mandatory tests, qualified clients, canary, exact-digest approval"
|
||
},
|
||
{
|
||
"from": "approved",
|
||
"to": "deprecated",
|
||
"gate": "Owner gives migration/replacement and retirement conditions"
|
||
},
|
||
{
|
||
"from": "deprecated",
|
||
"to": "retired",
|
||
"gate": "Consumers reconciled or explicit risk-approved cutover; no new invocation"
|
||
},
|
||
{
|
||
"from": "any active state",
|
||
"to": "quarantined",
|
||
"gate": "Incident or unsafe/unknown provenance; central deny immediately"
|
||
},
|
||
{
|
||
"from": "quarantined",
|
||
"to": "experimental",
|
||
"gate": "Reviewed correction; new immutable revision; requalify, never auto-restore approval"
|
||
}
|
||
],
|
||
"approval_separation": "Author cannot independently approve own release or edit its authoritative checks."
|
||
},
|
||
"quality_policy": {
|
||
"status": "Proposed pilot defaults requiring ratification before promotion; not literature-derived or currently approved.",
|
||
"mandatory_all_tiers": "100% mandatory deterministic contract, security, authorization and artifact checks pass; zero observed critical unauthorized effects. Insufficient coverage or missing approvals blocks promotion. Finite tests do not prove zero residual risk.",
|
||
"tiers": [
|
||
{
|
||
"id": "R1",
|
||
"scope": "Read-only/drafting; approved nonsensitive data; no external mutation",
|
||
"minimum_distinct_tasks": 30,
|
||
"trials_per_task_per_client": 3,
|
||
"minimum_artifact_success_percent": 90,
|
||
"selection_precision_floor_percent": 90,
|
||
"selection_recall_floor_percent": 90,
|
||
"maximum_review_interval_days": 90,
|
||
"review": "Owning team independent reviewer plus Quality Engineering rubric signoff"
|
||
},
|
||
{
|
||
"id": "R2",
|
||
"scope": "Proposed code/customer-facing artifact; human-reviewed, no privileged effect",
|
||
"minimum_distinct_tasks": 60,
|
||
"trials_per_task_per_client": 3,
|
||
"minimum_artifact_success_percent": 95,
|
||
"selection_precision_floor_percent": 95,
|
||
"selection_recall_floor_percent": 90,
|
||
"maximum_review_interval_days": 60,
|
||
"review": "R1 plus Architecture & Security data/tool review and owner acceptance"
|
||
},
|
||
{
|
||
"id": "R3",
|
||
"scope": "Privileged/production/financial/sensitive-personal workflows; excluded from initial autonomous scope",
|
||
"minimum_distinct_tasks": 100,
|
||
"trials_per_task_per_client": 5,
|
||
"minimum_artifact_success_percent": 99,
|
||
"selection_precision_floor_percent": 99,
|
||
"selection_recall_floor_percent": 95,
|
||
"maximum_review_interval_days": 30,
|
||
"review": "R2 plus explicit departmental authorization, Platform & IT recovery/kill-switch drill and operation-specific human approval"
|
||
}
|
||
],
|
||
"scoring": "Artifact success denominator is every planned executed trial, retaining failures and abandonment. Trials of same task are correlated: show per-task and per-client strata; do not inflate independent sample size. Selection floors use independent labels. Report 95% uncertainty intervals clustered by task for outcome comparisons; no regression versus current approved path beyond a preregistered owner-approved margin. Missing margin/precision rule prevents promotion.",
|
||
"required_cases": [
|
||
"positive trigger",
|
||
"near miss",
|
||
"negative trigger",
|
||
"prompt injection",
|
||
"malicious referenced script",
|
||
"missing prerequisite",
|
||
"permission denied",
|
||
"timeout",
|
||
"hook crash",
|
||
"malformed evidence",
|
||
"unsupported client",
|
||
"name collision",
|
||
"disabled skill",
|
||
"stale in-memory revision",
|
||
"context compaction",
|
||
"offline/reconnect",
|
||
"revoked revision",
|
||
"rollback",
|
||
"forged laptop attestation",
|
||
"concurrent coordinator duplicate",
|
||
"provider failure",
|
||
"human takeover"
|
||
]
|
||
},
|
||
"distribution": {
|
||
"default": "Reviewed vendored portable bundle plus qualified per-client adapters; private registry/distributor chosen in foundation.",
|
||
"manifest": [
|
||
"bundle_id",
|
||
"immutable_revision",
|
||
"digest",
|
||
"policy_revision",
|
||
"skills:[id,revision,digest]",
|
||
"adapters:[client,host,tested_version,revision,digest]",
|
||
"dependency_digests",
|
||
"approval_ref",
|
||
"issued_at",
|
||
"revocation_reference",
|
||
"rollback_bundle"
|
||
],
|
||
"locking": "Full commits/digests, no moving tags alone; atomic staging and effective-version readback including existing sessions.",
|
||
"offline": "Only explicitly permitted cached low-risk drafting with actual available dependencies; no guarantee of offline inference.",
|
||
"stale": "Deny centrally mediated privileged access until revalidated; low-risk grace requires explicit risk decision.",
|
||
"attestation_limit": "Client declaration is advisory, never sole proof of adherence or a bypass of central artifact gates."
|
||
},
|
||
"exceptions": {
|
||
"required": [
|
||
"exception_id",
|
||
"scope",
|
||
"skill_revision",
|
||
"client_tuple",
|
||
"reason",
|
||
"risk_owner",
|
||
"independent_approver",
|
||
"compensating_checks",
|
||
"starts_at",
|
||
"expires_at",
|
||
"affected_projects",
|
||
"rollback"
|
||
],
|
||
"expiry": "Fail closed for privileged access; no permanent silent bypass",
|
||
"nonwaivable": "Required human approval, protected release/payment/identity authority and independent audit."
|
||
},
|
||
"kpis": [
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Maturation horizon is workflow-specific and must be approved; no automatic abandonment on timeout.",
|
||
"id": "validated-task-success",
|
||
"name": "Validated task success",
|
||
"dimension": "outcome",
|
||
"definition": "Confirmed independently accepted tasks as a share of all eligible initiated tasks in a matured start cohort; a conservative confirmed-success share, not observed failure rate.",
|
||
"formula": {
|
||
"numerator": "Distinct matured task_ids with valid independent task.accepted evidence by as_of",
|
||
"denominator": "All distinct eligible task_ids started in window with started_at + preregistered task_observation_hours <= as_of",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Tasks in a preregistered workflow cohort, started in window; each counted once across retries, skills and clients.",
|
||
"exclusions": "Cancelled before start, sandbox/evaluation tasks and out-of-scope workflows; never post-start failures/abandonments.",
|
||
"source_events_tool": [
|
||
"task.started",
|
||
"task.accepted",
|
||
"task.failed",
|
||
"task.abandoned",
|
||
"task.cancelled",
|
||
"work-management acceptance record"
|
||
],
|
||
"owner": "Quality Engineering",
|
||
"cadence": "Weekly pilot; monthly departmental outcome review",
|
||
"gaming_risk": "Split easy tasks or omit starts; reconcile starts to approved backlog and audit acceptance rubrics.",
|
||
"missing_data_handling": "Unresolved/missing terminal outcomes remain in denominator with unknown count and known-outcome coverage; do not label them failed. Invalid/missing start excluded from calculation but flagged as orphan evidence.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 6,
|
||
"denominator": 10,
|
||
"result": 60.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Does not show execution success; precision depends on corpus prevalence.",
|
||
"id": "selection-precision",
|
||
"name": "Skill selection precision",
|
||
"dimension": "quality",
|
||
"definition": "Correct activations among activations on an independently labeled evaluation corpus.",
|
||
"formula": {
|
||
"numerator": "True-positive activations for the requested workflow",
|
||
"denominator": "True-positive plus false-positive activations",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "All candidate/client labeled evaluation task-trials with known activation and relevance.",
|
||
"exclusions": "Unlabeled/unknown activations excluded with coverage; no cherry-picked failed trials.",
|
||
"source_events_tool": [
|
||
"eval.selection",
|
||
"versioned labeled corpus and runner"
|
||
],
|
||
"owner": "Data & AI",
|
||
"cadence": "Each skill/client qualification",
|
||
"gaming_risk": "Choose easy labels or hide negative cases; freeze corpus and independently review labels.",
|
||
"missing_data_handling": "Unknown activation/relevance is unscorable; publish missing count and deny qualification if planned coverage incomplete.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 18,
|
||
"denominator": 20,
|
||
"result": 90.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Loading the right skill is not adherence.",
|
||
"id": "selection-recall",
|
||
"name": "Skill selection recall",
|
||
"dimension": "quality",
|
||
"definition": "Relevant task-trials in which the intended skill activates.",
|
||
"formula": {
|
||
"numerator": "True-positive activations",
|
||
"denominator": "True-positive plus false-negative task-trials",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Independently labeled relevant evaluation task-trials for a skill/client tuple.",
|
||
"exclusions": "Irrelevant tasks; missing relevance or activation reported as unscorable, not negative.",
|
||
"source_events_tool": [
|
||
"eval.selection",
|
||
"versioned labeled corpus and runner"
|
||
],
|
||
"owner": "Data & AI",
|
||
"cadence": "Each skill/client qualification",
|
||
"gaming_risk": "Overtrigger to maximize recall; always pair with precision and negative cases.",
|
||
"missing_data_handling": "Report evaluation coverage; null if no relevant labeled tasks.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 18,
|
||
"denominator": 24,
|
||
"result": 75.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Even verified file digests do not prove a private model followed the content.",
|
||
"id": "supported-bundle-coverage",
|
||
"name": "Supported bundle coverage",
|
||
"dimension": "risk",
|
||
"definition": "Inventoried active installations with verified effective approved, supported and nonrevoked bundle/adapter tuple.",
|
||
"formula": {
|
||
"numerator": "Eligible installation_ids with valid inventory evidence within approved freshness window and matching support matrix",
|
||
"denominator": "All expected active installation_ids in the managed pilot inventory",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Active installation inventory frozen for report, including invisible/offline entries until authoritative offboarding.",
|
||
"exclusions": "Verified decommissioned installations and nonpilot assets; lack of telemetry alone never excludes.",
|
||
"source_events_tool": [
|
||
"inventory.observed",
|
||
"catalog.published",
|
||
"catalog.revoked",
|
||
"device inventory"
|
||
],
|
||
"owner": "Platform & IT",
|
||
"cadence": "Weekly and at revocation",
|
||
"gaming_risk": "Remove silent devices or trust declared versions; reconcile managed inventory and separately label attestation trust.",
|
||
"missing_data_handling": "Unknown/stale versions count outside numerator, inside denominator. Show inventory discovery coverage versus eligible device roster; if roster unknown metric is provisional.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 8,
|
||
"denominator": 10,
|
||
"result": 80.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Self-report is not verified use; installation/loading is not use or benefit. Suppress small groups.",
|
||
"id": "useful-adoption",
|
||
"name": "Approved-workflow adoption",
|
||
"dimension": "adoption",
|
||
"definition": "Eligible participants with at least one approved-skill workflow use; usefulness assessed separately through accepted outcomes and feedback.",
|
||
"formula": {
|
||
"numerator": "Distinct eligible pseudonymous participants with workflow-linked skill.used on approved revision",
|
||
"denominator": "Distinct eligible participants in frozen invited cohort",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Consented or otherwise lawfully enrolled pilot cohort for the period; supported workflow available.",
|
||
"exclusions": "Outside pilot and formally recorded leave before period; nonresponders remain eligible, not excluded.",
|
||
"source_events_tool": [
|
||
"skill.used",
|
||
"cohort.snapshot",
|
||
"optional qualified client adapter or declaration"
|
||
],
|
||
"owner": "Data & AI",
|
||
"cadence": "Monthly",
|
||
"gaming_risk": "Invocation quotas and coercion; no target usage count, split server/client/self-report origins.",
|
||
"missing_data_handling": "Show observed lower bound and telemetry/response coverage; missing usage unknown rather than non-use.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 7,
|
||
"denominator": 10,
|
||
"result": 70.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Not PR cycle time, effort saved, or a complete view of unfinished changes. p90 uses nearest-rank sorted sample.",
|
||
"id": "change-lead-time",
|
||
"name": "Change lead time",
|
||
"dimension": "flow",
|
||
"definition": "Arithmetic mean commit-to-first-successful-production-deployment duration; also publish median/p90 and sample count.",
|
||
"formula": {
|
||
"numerator": "Sum of elapsed hours from committed_at to first successful production deployment for each service+commit",
|
||
"denominator": "Count of eligible service+commit pairs with valid mappings",
|
||
"expression": "N / D",
|
||
"aggregation": "mean"
|
||
},
|
||
"unit": "hours/change",
|
||
"eligible_population": "Changes first successfully deployed to a service in reporting window; rollout retries collapse to same logical deployment.",
|
||
"exclusions": "Nonproduction, duplicates, no-op deployment with no new commits; missing mappings shown as coverage loss.",
|
||
"source_events_tool": [
|
||
"change.committed",
|
||
"deployment.completed",
|
||
"VCS immutable commit timestamps and release manifest"
|
||
],
|
||
"owner": "Platform & IT",
|
||
"cadence": "Weekly per service",
|
||
"gaming_risk": "Timestamp rewriting and cherry-picking fast work; lock change mapping and show undeployed backlog age.",
|
||
"missing_data_handling": "Missing commit mapping or negative/skewed duration excluded from numeric sample, reported explicitly; no zero substitution.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 120,
|
||
"denominator": 10,
|
||
"result": 12.0,
|
||
"unit": "hours/change"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. More deployments alone do not establish better performance; affected unsuccessful changes remain in change-fail population.",
|
||
"id": "deployment-frequency",
|
||
"name": "Deployment frequency",
|
||
"dimension": "flow",
|
||
"definition": "Distinct successful logical production deployments per complete service-day of observation.",
|
||
"formula": {
|
||
"numerator": "Distinct service_id+deployment_id successful production deployments in window",
|
||
"denominator": "Observed service-days in window",
|
||
"expression": "N / D",
|
||
"aggregation": "ratio"
|
||
},
|
||
"unit": "deployments/service-day",
|
||
"eligible_population": "Declared production services and complete UTC observation days; report each service before pooled views.",
|
||
"exclusions": "Test/staging, failed attempts without production effect, duplicate callbacks and retries of the same rollout.",
|
||
"source_events_tool": [
|
||
"deployment.completed",
|
||
"authoritative deployment service"
|
||
],
|
||
"owner": "Platform & IT",
|
||
"cadence": "Weekly per service",
|
||
"gaming_risk": "Split deployments without value; pair with success, rework and failure measures.",
|
||
"missing_data_handling": "Missing feed periods not zero; report coverage and no unqualified overall rate when period incomplete.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 14,
|
||
"denominator": 7,
|
||
"result": 2.0,
|
||
"unit": "deployments/service-day"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Define immediate intervention and follow-up horizon before pilot; no universal incident-to-AI attribution.",
|
||
"id": "change-fail-rate",
|
||
"name": "Change fail rate",
|
||
"dimension": "quality",
|
||
"definition": "Production deployment effects requiring unplanned remediation, rollback or hotfix, not solely outage declarations.",
|
||
"formula": {
|
||
"numerator": "Distinct matured production deployment_ids linked to qualifying immediate intervention",
|
||
"denominator": "All distinct production deployment_ids that reached production in the matured deployment cohort",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Deployments with production effect during window and approved intervention follow-up horizon elapsed.",
|
||
"exclusions": "Failed preproduction attempts, duplicate callbacks and no-effect retries; failures after production effect stay included.",
|
||
"source_events_tool": [
|
||
"deployment.completed",
|
||
"deployment.failed",
|
||
"incident.linked",
|
||
"release and incident records"
|
||
],
|
||
"owner": "Quality Engineering",
|
||
"cadence": "Weekly; finalized after horizon",
|
||
"gaming_risk": "Avoid incident links or relabel intervention planned; reconcile rollbacks/hotfixes independently.",
|
||
"missing_data_handling": "Known failures / all eligible deployments is a lower bound where incident linkage incomplete; label provisional and report unknown links.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 2,
|
||
"denominator": 20,
|
||
"result": 10.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Not all-incident MTTR; recovered-only sample has survivorship bias. p90 nearest rank.",
|
||
"id": "failed-deployment-recovery-time",
|
||
"name": "Failed deployment recovery time",
|
||
"dimension": "flow",
|
||
"definition": "Mean elapsed time from production deployment failure onset to service restoration, with median/p90 and unresolved count/age.",
|
||
"formula": {
|
||
"numerator": "Sum of recovery_at minus failure_started_at in hours for recovered eligible deployment failures",
|
||
"denominator": "Count of recovered eligible deployment failures with valid timestamps",
|
||
"expression": "N / D",
|
||
"aggregation": "mean"
|
||
},
|
||
"unit": "hours/failure",
|
||
"eligible_population": "Production deployment failures beginning in window; latest verified restoration if a recovery is rescinded.",
|
||
"exclusions": "Nondeployment incidents; invalid timestamp pairs reported missing, not treated as instantaneous recovery.",
|
||
"source_events_tool": [
|
||
"deployment.failed",
|
||
"service.restored",
|
||
"incident service"
|
||
],
|
||
"owner": "Platform & IT",
|
||
"cadence": "Weekly with unresolved failures",
|
||
"gaming_risk": "Premature recovery declarations or omit hard cases; require operational readback and carry unresolved ages.",
|
||
"missing_data_handling": "Unresolved failures reported separately and excluded only from recovered-duration distribution; prevent declaring improvement without their visibility.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 9,
|
||
"denominator": 3,
|
||
"result": 3.0,
|
||
"unit": "hours/failure"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Distinct from failure rate: one failed deployment can lead to multiple separate rework deployments.",
|
||
"id": "deployment-rework-rate",
|
||
"name": "Deployment rework rate",
|
||
"dimension": "quality",
|
||
"definition": "Unplanned production deployment effects triggered by a production incident as a share of deployment effects.",
|
||
"formula": {
|
||
"numerator": "Distinct incident-linked deployment_ids classified unplanned remediation",
|
||
"denominator": "All distinct production deployment_ids with production effect in window",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Production deployment effects in window with immutable planned/unplanned classification and incident linkage.",
|
||
"exclusions": "Nonproduction, no-effect retries and duplicates; do not exclude unclassified changes silently.",
|
||
"source_events_tool": [
|
||
"deployment.completed",
|
||
"incident.linked",
|
||
"release intent classification"
|
||
],
|
||
"owner": "Platform & IT",
|
||
"cadence": "Weekly",
|
||
"gaming_risk": "Call hotfixes planned; freeze reason and independently reconcile incident/deployment links.",
|
||
"missing_data_handling": "Unknown classification stays denominator; observed numerator is lower bound until reconciliation; show classification coverage.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 3,
|
||
"denominator": 20,
|
||
"result": 15.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Counts affected changes, not raw defects per change; no presumption defects are AI-caused.",
|
||
"id": "escaped-quality-rate",
|
||
"name": "Changes with escaped quality issues",
|
||
"dimension": "quality",
|
||
"definition": "Accepted changes with at least one confirmed post-acceptance defect during approved follow-up, split by severity.",
|
||
"formula": {
|
||
"numerator": "Distinct eligible change_ids with one or more linked confirmed defects in follow-up",
|
||
"denominator": "Accepted change_ids with completed predeclared follow-up horizon",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Accepted changes in acceptance window whose follow-up is complete by as_of.",
|
||
"exclusions": "Immature changes shown separately; duplicates/unconfirmed reports excluded but counted as pending triage.",
|
||
"source_events_tool": [
|
||
"change.accepted",
|
||
"defect.confirmed",
|
||
"defect tracking and acceptance evidence"
|
||
],
|
||
"owner": "Quality Engineering",
|
||
"cadence": "Monthly mature cohorts",
|
||
"gaming_risk": "Shorten follow-up or downgrade severity; freeze horizon and rubric and independently sample defects.",
|
||
"missing_data_handling": "Unknown linkage is a coverage warning; no fabricated clean outcome. Show absolute defect and severe incident counts as well.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 2,
|
||
"denominator": 20,
|
||
"result": 10.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Blocked attempts are not actual effects; any critical effect blocks promotion regardless of aggregate rate.",
|
||
"id": "policy-effects",
|
||
"name": "Confirmed policy effects",
|
||
"dimension": "risk",
|
||
"definition": "Eligible evaluated attempts with an actual confirmed prohibited effect; report attempts, blocked attempts and effects separately.",
|
||
"formula": {
|
||
"numerator": "Distinct attempt_ids with confirmed prohibited action effect",
|
||
"denominator": "All evaluated eligible attempt_ids including failed and abandoned runs",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Governed skill trials or supported workflow attempts, reported separately for eval and live populations.",
|
||
"exclusions": "Out-of-scope runs; detection gaps remain coverage issues, not exclusions from known initiated population.",
|
||
"source_events_tool": [
|
||
"attempt.started",
|
||
"policy.violation",
|
||
"gate.blocked",
|
||
"security audit and incident triage"
|
||
],
|
||
"owner": "Architecture & Security",
|
||
"cadence": "Immediate incident action; weekly aggregates",
|
||
"gaming_risk": "Hide incidents or inflate harmless trials; absolute severity counts and fixed task mix accompany rate.",
|
||
"missing_data_handling": "Unknown outcomes remain visible; zero detected effects never proves safety. Missing audits make rate provisional.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 1,
|
||
"denominator": 100,
|
||
"result": 1.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Allocation and sampled labor carry uncertainty; no ROI without independently measured business benefit.",
|
||
"id": "cost-per-accepted-outcome",
|
||
"name": "Cost per accepted outcome",
|
||
"dimension": "cost",
|
||
"definition": "Allocated total workflow cost including unsuccessful attempts, licenses, compute, tooling and human review/rework divided by accepted outcomes.",
|
||
"formula": {
|
||
"numerator": "Sum allocated costs in one approved reporting currency for all tasks/attempts in matured start cohort",
|
||
"denominator": "Distinct independently accepted tasks in same cohort",
|
||
"expression": "N / D",
|
||
"aggregation": "ratio"
|
||
},
|
||
"unit": "currency/accepted outcome",
|
||
"eligible_population": "Matured eligible started-task cohort, with all attempt costs through as_of and allocated shared period costs.",
|
||
"exclusions": "Unrelated projects and documented taxes/transfers outside chosen accounting basis; never failed/abandoned attempts.",
|
||
"source_events_tool": [
|
||
"cost.recorded",
|
||
"task.accepted",
|
||
"finance invoices, usage ledger, sampled effort with approved rates"
|
||
],
|
||
"owner": "Finance & Accounting Squad",
|
||
"cadence": "Monthly with late invoice restatement",
|
||
"gaming_risk": "Count tokens only or hide labor/failures; reconcile invoice and allocation totals without double counting.",
|
||
"missing_data_handling": "Missing cost buckets produce partial estimate plus coverage, not a total. Mixed currency requires dated approved conversion; zero accepted outcomes = null with full cost shown.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 600,
|
||
"denominator": 6,
|
||
"result": 100.0,
|
||
"unit": "currency/accepted outcome"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Perception and selection bias; not measured time savings. No composite productivity index.",
|
||
"id": "human-experience",
|
||
"name": "Verification confidence and cognitive load",
|
||
"dimension": "experience",
|
||
"definition": "Three separate anonymous 1–5 item distributions: confidence in verifying output, manageable workload, usefulness. Primary item: verification confidence mean; report full distributions and response rate.",
|
||
"formula": {
|
||
"numerator": "Sum valid 1–5 verification-confidence responses, one per participant per survey wave",
|
||
"denominator": "Count valid responses to that item",
|
||
"expression": "N / D",
|
||
"aggregation": "mean"
|
||
},
|
||
"unit": "points (1–5)",
|
||
"eligible_population": "Invited eligible cohort at fixed survey wave; voluntary responses.",
|
||
"exclusions": "Invalid/out-of-range answers and superseded duplicate submission; nonresponders stay in response-rate denominator.",
|
||
"source_events_tool": [
|
||
"survey.response",
|
||
"anonymous survey with single-use token and aggregate invitation count"
|
||
],
|
||
"owner": "People & Administration Squad",
|
||
"cadence": "After pilot workflow cycle; periodic follow-up",
|
||
"gaming_risk": "Pressure to respond positively; no manager access to individual responses, avoid identifying free text.",
|
||
"missing_data_handling": "Report valid response count / invited count and item nonresponse; no imputation. Suppress people cohorts below five.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 36,
|
||
"denominator": 10,
|
||
"result": 3.6,
|
||
"unit": "points (1–5)"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. A current review is not proof of effectiveness; consumer inventory may miss shadow copies.",
|
||
"id": "maintenance-health",
|
||
"name": "Skills within substantive review window",
|
||
"dimension": "risk",
|
||
"definition": "Active approved skill revisions with an accountable owner and completed substantive review not past due.",
|
||
"formula": {
|
||
"numerator": "Distinct active approved skill revisions with current owner, review evidence and review_due_at >= as_of",
|
||
"denominator": "All active approved skill revisions in catalog snapshot",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "Approved revisions currently supported for use at as_of.",
|
||
"exclusions": "Experimental and retired revisions; deprecated/revoked in-use counts reported independently, not hidden.",
|
||
"source_events_tool": [
|
||
"catalog.published",
|
||
"catalog.reviewed",
|
||
"catalog.deprecated",
|
||
"catalog.revoked",
|
||
"version inventory"
|
||
],
|
||
"owner": "Data & AI",
|
||
"cadence": "Monthly and each release",
|
||
"gaming_risk": "Rubber-stamp review dates; require linked regression/security/usefulness evidence.",
|
||
"missing_data_handling": "Missing owner/date/evidence outside numerator; show ownerless, deprecated-in-use and revoked-in-use absolute counts.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 9,
|
||
"denominator": 12,
|
||
"result": 75.0,
|
||
"unit": "%"
|
||
}
|
||
},
|
||
{
|
||
"reporting_rules": "Predeclare population and UTC half-open window [start,end); report as_of and calculation revision. Zero denominator = null/not available. Never drop missing data silently. Report sample size, coverage and cohort/task mix.",
|
||
"baseline_collection": "Prospective current-practice/no-skill baseline using identical eligibility and acceptance rules; finish a comparable workflow cycle and preregistered sample/precision requirement. Record concurrent process, client and model changes. No observed baseline exists in this handoff.",
|
||
"privacy": "No raw prompts, code, secrets or individual rankings. Team/service aggregate; people cohorts below five suppressed and overlapping slices restricted. Actor IDs only where necessary under approved collection basis.",
|
||
"evidence_limitations": "Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Domain owner signs acceptance: Operations service accuracy; Marketing claims/rights; Finance reconciliation accuracy; HR policy correctness; Product usability/value. Not revenue uplift.",
|
||
"id": "department-outcome",
|
||
"name": "Workflow-specific business acceptance",
|
||
"dimension": "outcome",
|
||
"definition": "Independently accepted domain outcomes against a predeclared business rubric, separate from technical completion.",
|
||
"formula": {
|
||
"numerator": "Eligible completed-observation task_ids meeting the department-specific outcome rubric",
|
||
"denominator": "All eligible initiated workflow task_ids with matured business-outcome horizon",
|
||
"expression": "N / D * 100",
|
||
"aggregation": "percent"
|
||
},
|
||
"unit": "%",
|
||
"eligible_population": "One named departmental workflow and predeclared start cohort; never pool unlike rubrics.",
|
||
"exclusions": "Cancelled-before-start and unrelated workflows; unsuccessful or abandoned eligible work remains.",
|
||
"source_events_tool": [
|
||
"task.started",
|
||
"business.accepted",
|
||
"departmental case/content/reconciliation/onboarding review record"
|
||
],
|
||
"owner": "Product Management",
|
||
"cadence": "Each pilot cycle with owning department",
|
||
"gaming_risk": "Weaken rubric or count drafts as customer benefit; lock rubric and independently audit acceptance.",
|
||
"missing_data_handling": "Unknown follow-up stays denominator with unresolved count; no claim actual business outcome changed without source evidence.",
|
||
"illustrative_example": {
|
||
"label": "SYNTHETIC — illustrative, not Eldorado observations",
|
||
"numerator": 8,
|
||
"denominator": 10,
|
||
"result": 80.0,
|
||
"unit": "%"
|
||
}
|
||
}
|
||
],
|
||
"event_contract": {
|
||
"schema_version": "1.0",
|
||
"required_envelope": {
|
||
"event_id": "Stable producer event ID; never regenerate on transport retry",
|
||
"schema_version": "1.0",
|
||
"event_type": "enum below",
|
||
"occurred_at": "UTC RFC3339",
|
||
"received_at": "Collector UTC RFC3339",
|
||
"producer": "Stable registered producer ID",
|
||
"project_id": "Opaque project or cohort scope",
|
||
"measurement_origin": "server|client|self-report|synthetic",
|
||
"collection_basis": "Approved purpose/basis reference; synthetic for fixtures",
|
||
"evidence_ref": "Opaque restricted record ref, not secret-bearing URL"
|
||
},
|
||
"nullable_context": {
|
||
"task_id": "Required for task/attempt/skill events",
|
||
"attempt_id": "Required for attempt/skill/policy events",
|
||
"actor_id": "Pseudonymous; only for approved adoption association",
|
||
"client": "{name,host,version,os}; null plus reason when unavailable",
|
||
"model": "{id,version}; hidden provider version must stay null",
|
||
"skill_refs": "Array of {skill_id,revision,digest,adapter_revision}; exact observed tuple, not inferred latest",
|
||
"policy_revision": "Exact enforced policy digest when gate/approval event",
|
||
"repo_commit": "Full immutable commit when change/gate/release event",
|
||
"outcome": "typed status, never free-form success inference",
|
||
"reason_code": "controlled vocabulary",
|
||
"missing_reasons": "Map field -> not_exposed|not_collected|not_applicable|lost|untrusted_clock"
|
||
},
|
||
"event_required_payload": {
|
||
"task.started": [
|
||
"task_id",
|
||
"workflow_id",
|
||
"eligibility_rule_revision",
|
||
"started_at",
|
||
"task_observation_hours"
|
||
],
|
||
"task.accepted": [
|
||
"task_id",
|
||
"acceptance_rubric_revision",
|
||
"independent_reviewer_ref",
|
||
"artifact_ref"
|
||
],
|
||
"task.failed": [
|
||
"task_id",
|
||
"reason_code"
|
||
],
|
||
"task.abandoned": [
|
||
"task_id",
|
||
"owner_reconciliation_ref",
|
||
"reason_code"
|
||
],
|
||
"task.cancelled": [
|
||
"task_id",
|
||
"before_start",
|
||
"reason_code"
|
||
],
|
||
"attempt.started": [
|
||
"task_id",
|
||
"attempt_id"
|
||
],
|
||
"attempt.failed": [
|
||
"task_id",
|
||
"attempt_id",
|
||
"reason_code"
|
||
],
|
||
"attempt.abandoned": [
|
||
"task_id",
|
||
"attempt_id",
|
||
"owner_reconciliation_ref",
|
||
"reason_code"
|
||
],
|
||
"skill.used": [
|
||
"task_id",
|
||
"attempt_id",
|
||
"skill_refs",
|
||
"client",
|
||
"origin_confidence"
|
||
],
|
||
"skill.loaded": [
|
||
"task_id",
|
||
"attempt_id",
|
||
"skill_refs",
|
||
"client"
|
||
],
|
||
"eval.selection": [
|
||
"eval_id",
|
||
"trial_id",
|
||
"task_case_id",
|
||
"relevant",
|
||
"activated",
|
||
"skill_refs",
|
||
"client"
|
||
],
|
||
"inventory.observed": [
|
||
"installation_id",
|
||
"bundle_digest",
|
||
"adapter_revision",
|
||
"verified_at",
|
||
"verification_method",
|
||
"support_status"
|
||
],
|
||
"cohort.snapshot": [
|
||
"cohort_id",
|
||
"eligibility_rule_revision",
|
||
"eligible_actor_ids_or_restricted_roster_ref"
|
||
],
|
||
"change.committed": [
|
||
"change_id",
|
||
"service_id",
|
||
"repo_commit",
|
||
"committed_at"
|
||
],
|
||
"change.accepted": [
|
||
"change_id",
|
||
"accepted_at",
|
||
"followup_hours",
|
||
"independent_reviewer_ref"
|
||
],
|
||
"deployment.completed": [
|
||
"deployment_id",
|
||
"service_id",
|
||
"environment",
|
||
"production_effect",
|
||
"succeeded",
|
||
"deployed_at",
|
||
"repo_commits",
|
||
"planned",
|
||
"incident_id_or_null"
|
||
],
|
||
"deployment.failed": [
|
||
"deployment_id",
|
||
"service_id",
|
||
"production_effect",
|
||
"failure_started_at",
|
||
"incident_id"
|
||
],
|
||
"service.restored": [
|
||
"deployment_id",
|
||
"incident_id",
|
||
"restored_at",
|
||
"verification_ref"
|
||
],
|
||
"incident.linked": [
|
||
"incident_id",
|
||
"deployment_id",
|
||
"link_kind",
|
||
"intervention_type",
|
||
"confirmed_by"
|
||
],
|
||
"defect.confirmed": [
|
||
"defect_id",
|
||
"change_id",
|
||
"severity",
|
||
"detected_at",
|
||
"confirmed_at"
|
||
],
|
||
"policy.violation": [
|
||
"attempt_id",
|
||
"task_id",
|
||
"effect_id",
|
||
"severity",
|
||
"actual_effect",
|
||
"confirmed_at"
|
||
],
|
||
"gate.blocked": [
|
||
"attempt_id",
|
||
"task_id",
|
||
"gate_id",
|
||
"policy_revision",
|
||
"artifact_ref"
|
||
],
|
||
"cost.recorded": [
|
||
"cost_id",
|
||
"cohort_id",
|
||
"task_id_or_shared_pool",
|
||
"amount",
|
||
"currency",
|
||
"basis",
|
||
"allocation_rule_revision"
|
||
],
|
||
"survey.response": [
|
||
"wave_id",
|
||
"anonymous_response_token",
|
||
"item_id",
|
||
"score"
|
||
],
|
||
"business.accepted": [
|
||
"task_id",
|
||
"department",
|
||
"business_rubric_revision",
|
||
"independent_reviewer_ref"
|
||
],
|
||
"catalog.published": [
|
||
"skill_id",
|
||
"revision",
|
||
"digest",
|
||
"owner",
|
||
"approval_ref",
|
||
"review_due_at"
|
||
],
|
||
"catalog.reviewed": [
|
||
"skill_id",
|
||
"revision",
|
||
"review_evidence_ref",
|
||
"review_due_at"
|
||
],
|
||
"catalog.deprecated": [
|
||
"skill_id",
|
||
"revision",
|
||
"replacement",
|
||
"retirement_condition"
|
||
],
|
||
"catalog.revoked": [
|
||
"skill_id",
|
||
"revision",
|
||
"reason_code",
|
||
"revocation_at"
|
||
]
|
||
},
|
||
"deduplication": "Primary transport key (producer,event_id); compare immutable producer payload, excluding collector received_at. Identical redelivery keeps earliest receipt and records later receipts separately; conflicting producer payload quarantined and affected aggregate marked incomplete. Semantic keys task_id, attempt_id, service_id+deployment_id, defect_id, cost_id, effect_id prevent cross-feed double counting.",
|
||
"ordering": "Keep occurrence and receipt times; immutable events. Reject impossible timestamp order; flag skew against registered tolerance. Trusted server timestamps dominate where available. Late events recompute versioned aggregates with as_of and revision.",
|
||
"outcome_resolution": "Single task across attempts. Accepted requires independent evidence and supersedes attempt failures; conflicting terminal task records require reconciliation, not last-write-wins. Missing terminal event remains unknown; explicit owner evidence needed to mark abandonment. Post-start cancellation remains nonsuccess in matured denominator.",
|
||
"association": "Many-to-many task/attempt/skill/client tuples; do not attribute entire outcome or cost to every skill. Mixed-version tasks form mixed cohort; unknown versions form unknown cohort, not latest.",
|
||
"minimal_collection": "Start with authoritative server events and catalog/inventory; optional local events only after client qualification and privacy approval. Not every event family is needed for first pilot."
|
||
},
|
||
"privacy_proposal": {
|
||
"approval_status": "pending responsible-owner review before collection",
|
||
"raw_analytics_retention_days": 30,
|
||
"aggregate_retention_months": 12,
|
||
"minimum_people_cohort": 5,
|
||
"forbidden_default_payload": [
|
||
"raw prompt",
|
||
"source code",
|
||
"screenshots",
|
||
"customer case text",
|
||
"secret",
|
||
"raw tool arguments"
|
||
],
|
||
"access": "Platform & IT operates restricted collection; Data & AI sees minimized aggregate/analysis data; Architecture & Security incident evidence access; People & Administration Squad controls survey privacy; department owners only appropriate aggregates. Access logged.",
|
||
"deletion": "Document subject access/deletion request path; delete or irreversibly aggregate raw identifiers at expiry; approved incident/legal holds isolated with explicit owner/expiry; catalog/approval audit retention decided separately."
|
||
},
|
||
"prototype_test_plan": {
|
||
"reference": "sample_calculations.py; run python3 artifacts/ai-adoption/sample_calculations.py",
|
||
"implementation": "Port into repository standard test runner with JSON synthetic fixtures and CLI; reference is not production analytics.",
|
||
"cases": [
|
||
"identical duplicate",
|
||
"conflicting duplicate",
|
||
"failed and abandoned retained",
|
||
"retry counts one task",
|
||
"immature cohort excluded",
|
||
"zero denominator returns null",
|
||
"missing terminal remains unknown",
|
||
"missing start is orphan",
|
||
"late acceptance restates",
|
||
"unknown version stays unknown",
|
||
"mixed versions not double attributed",
|
||
"semantic duplicate across feeds",
|
||
"invalid timestamp fails",
|
||
"deployment retry dedup",
|
||
"immature defect follow-up",
|
||
"cost of failed attempt included",
|
||
"policy blocked vs actual effect",
|
||
"survey small cohort suppression"
|
||
],
|
||
"required_outputs": [
|
||
"label",
|
||
"as_of",
|
||
"calculation_version",
|
||
"numerator",
|
||
"denominator",
|
||
"unit",
|
||
"value_or_null",
|
||
"unknown_count",
|
||
"exclusions",
|
||
"coverage",
|
||
"warnings"
|
||
]
|
||
},
|
||
"evidence_map": [
|
||
{
|
||
"recommendation": "portable-contract-adapters",
|
||
"source_ids": [
|
||
1,
|
||
2,
|
||
4,
|
||
8,
|
||
9,
|
||
21,
|
||
26
|
||
]
|
||
},
|
||
{
|
||
"recommendation": "independent-authority",
|
||
"source_ids": [
|
||
3,
|
||
5,
|
||
6,
|
||
7,
|
||
15,
|
||
23
|
||
]
|
||
},
|
||
{
|
||
"recommendation": "layered-evaluation",
|
||
"source_ids": [
|
||
11,
|
||
15
|
||
]
|
||
},
|
||
{
|
||
"recommendation": "balanced-measurement",
|
||
"source_ids": [
|
||
12,
|
||
24
|
||
]
|
||
},
|
||
{
|
||
"recommendation": "no-unvalidated-roi",
|
||
"source_ids": [
|
||
16,
|
||
17,
|
||
18,
|
||
22
|
||
]
|
||
},
|
||
{
|
||
"recommendation": "minimal-telemetry",
|
||
"source_ids": [
|
||
19,
|
||
20,
|
||
23
|
||
]
|
||
},
|
||
{
|
||
"recommendation": "lifecycle-risk",
|
||
"source_ids": [
|
||
13,
|
||
14,
|
||
25
|
||
]
|
||
}
|
||
],
|
||
"open_decisions_reference": "strategy-blueprint.md — Internal evidence mapping and open decisions",
|
||
"publication_boundary": "Evidence mapping, research and open decisions stay outside reader-facing docs/static/search; proposed quality/privacy values must be visibly target settings, never current installation claims."
|
||
}
|