diff --git a/docs/core/maintain.md b/docs/core/maintain.md index bda4d55..449b429 100644 --- a/docs/core/maintain.md +++ b/docs/core/maintain.md @@ -28,10 +28,10 @@ Include `title` and `description` frontmatter. Place each page in its owning sec | Contribution | `docs/contribution/` | `sidebars/contribution.ts` | | Portal maintenance and reference | `docs/core/` | `sidebars.ts` | -Keep these content roots disjoint. Preserve existing URLs when moving pages. Specific AI initiatives and organizational reporting lines remain subject to agreement. +Keep these content roots disjoint. Preserve existing URLs unless a URL migration is explicitly approved; provide precise redirects for approved moves. Metric details live on individual canonical pages directly under Measurement, alongside shared collection safeguards and the synthetic example. Specific AI initiatives and organizational reporting lines remain subject to agreement. Every published documentation page must have a navigable sidebar entry. Nested references are welcome; hidden documentation is not. Linked category overviews count as entries, but an in-page anchor is not a separate document. Portal-maintenance pages keep their existing `/docs/` routes and dedicated sidebar, reachable from Contribution → Portal maintenance. -`pnpm verify:build` compares all Docusaurus-generated document IDs/routes with resolved sidebar trees (including linked category overviews), checks reachability from the navbar, rejects missing/duplicate entries and unclassified HTML routes, and checks every document’s local links and anchors. The inventory is written to `.cache/sidebar-coverage.json`. Only the homepage, not-found page, search-results UI, legacy Strategy redirect and standalone diagram viewers are non-document HTML routes; no documentation page is exempt. +`pnpm verify:build` compares all Docusaurus-generated document IDs/routes with resolved sidebar trees (including linked category overviews), checks reachability from the navbar, rejects missing/duplicate entries and unclassified HTML routes, and checks every document’s local links and anchors. The inventory is written to `.cache/sidebar-coverage.json`. Only the homepage, not-found page, search-results UI, legacy Strategy redirect, exact legacy measurement-contracts hash-aware redirect and standalone diagram viewers are non-document HTML routes; no documentation page is exempt. The metric migration manifest in `research/ai-adoption/measurement-migration.json` records old anchors, canonical IDs/URLs, implementation status and preserved-contract hashes. ## Validate the change diff --git a/docs/strategy/measurement/approved-workflow-adoption.mdx b/docs/strategy/measurement/approved-workflow-adoption.mdx new file mode 100644 index 0000000..4a0780e --- /dev/null +++ b/docs/strategy/measurement/approved-workflow-adoption.mdx @@ -0,0 +1,36 @@ +--- +id: approved-workflow-adoption +slug: /measurement/approved-workflow-adoption +title: Approved-workflow adoption +description: Definition, collection evidence and limitations for approved-workflow adoption. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Unimplemented metric — no live telemetry.** This is a prospective contract, not an implemented calculation or collector. + +{/* contract:start */} +Eligible participants with at least one approved-skill workflow use; usefulness assessed separately through accepted outcomes and feedback. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct eligible pseudonymous participants with workflow-linked skill.used on approved revision | +| Denominator | Distinct eligible participants in frozen invited cohort | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Consented or otherwise lawfully enrolled pilot cohort for the period; supported workflow available. | +| Exclusions | Outside pilot and formally recorded leave before period; nonresponders remain eligible, not excluded. | +| Events and source | skill.used; cohort.snapshot; optional qualified client adapter or declaration | +| Owner / cadence | Data & AI / Monthly | +| Missing evidence | Show observed lower bound and telemetry/response coverage; missing usage unknown rather than non-use. | +| Gaming and limits | Invocation quotas and coercion; no target usage count, split server/client/self-report origins. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Self-report is not verified use; installation/loading is not use or benefit. Suppress small groups. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/change-fail-rate.mdx b/docs/strategy/measurement/change-fail-rate.mdx new file mode 100644 index 0000000..bf0ef0f --- /dev/null +++ b/docs/strategy/measurement/change-fail-rate.mdx @@ -0,0 +1,36 @@ +--- +id: change-fail-rate +slug: /measurement/change-fail-rate +title: Change fail rate +description: Definition, collection evidence and limitations for change fail rate. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Production deployment effects requiring unplanned remediation, rollback or hotfix, not solely outage declarations. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct matured production deployment_ids linked to qualifying immediate intervention | +| Denominator | All distinct production deployment_ids that reached production in the matured deployment cohort | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Deployments with production effect during window and approved intervention follow-up horizon elapsed. | +| Exclusions | Failed preproduction attempts, duplicate callbacks and no-effect retries; failures after production effect stay included. | +| Events and source | deployment.completed; deployment.failed; incident.linked; release and incident records | +| Owner / cadence | Quality Engineering / Weekly; finalized after horizon | +| Missing evidence | Known failures / all eligible deployments is a lower bound where incident linkage incomplete; label provisional and report unknown links. | +| Gaming and limits | Avoid incident links or relabel intervention planned; reconcile rollbacks/hotfixes independently. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Define immediate intervention and follow-up horizon before pilot; no universal incident-to-AI attribution. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/change-lead-time.mdx b/docs/strategy/measurement/change-lead-time.mdx new file mode 100644 index 0000000..f0049d0 --- /dev/null +++ b/docs/strategy/measurement/change-lead-time.mdx @@ -0,0 +1,36 @@ +--- +id: change-lead-time +slug: /measurement/change-lead-time +title: Change lead time +description: Definition, collection evidence and limitations for change lead time. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Unimplemented metric — no live telemetry.** This is a prospective contract, not an implemented calculation or collector. + +{/* contract:start */} +Arithmetic mean commit-to-first-successful-production-deployment duration; also publish median/p90 and sample count. + +| Contract | Definition | +| --- | --- | +| Numerator | Sum of elapsed hours from committed_at to first successful production deployment for each service+commit | +| Denominator | Count of eligible service+commit pairs with valid mappings | +| Expression | N / D | +| Aggregation | mean | +| Unit | hours/change | +| Eligible population | Changes first successfully deployed to a service in reporting window; rollout retries collapse to same logical deployment. | +| Exclusions | Nonproduction, duplicates, no-op deployment with no new commits; missing mappings shown as coverage loss. | +| Events and source | change.committed; deployment.completed; VCS immutable commit timestamps and release manifest | +| Owner / cadence | Platform & IT / Weekly per service | +| Missing evidence | Missing commit mapping or negative/skewed duration excluded from numeric sample, reported explicitly; no zero substitution. | +| Gaming and limits | Timestamp rewriting and cherry-picking fast work; lock change mapping and show undeployed backlog age. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Not PR cycle time, effort saved, or a complete view of unfinished changes. p90 uses nearest-rank sorted sample. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/changes-with-escaped-quality-issues.mdx b/docs/strategy/measurement/changes-with-escaped-quality-issues.mdx new file mode 100644 index 0000000..fa42f29 --- /dev/null +++ b/docs/strategy/measurement/changes-with-escaped-quality-issues.mdx @@ -0,0 +1,36 @@ +--- +id: changes-with-escaped-quality-issues +slug: /measurement/changes-with-escaped-quality-issues +title: Changes with escaped quality issues +description: Definition, collection evidence and limitations for changes with escaped quality issues. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Accepted changes with at least one confirmed post-acceptance defect during approved follow-up, split by severity. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct eligible change_ids with one or more linked confirmed defects in follow-up | +| Denominator | Accepted change_ids with completed predeclared follow-up horizon | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Accepted changes in acceptance window whose follow-up is complete by as_of. | +| Exclusions | Immature changes shown separately; duplicates/unconfirmed reports excluded but counted as pending triage. | +| Events and source | change.accepted; defect.confirmed; defect tracking and acceptance evidence | +| Owner / cadence | Quality Engineering / Monthly mature cohorts | +| Missing evidence | Unknown linkage is a coverage warning; no fabricated clean outcome. Show absolute defect and severe incident counts as well. | +| Gaming and limits | Shorten follow-up or downgrade severity; freeze horizon and rubric and independently sample defects. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Counts affected changes, not raw defects per change; no presumption defects are AI-caused. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/collection-and-safeguards.mdx b/docs/strategy/measurement/collection-and-safeguards.mdx new file mode 100644 index 0000000..13c8e8e --- /dev/null +++ b/docs/strategy/measurement/collection-and-safeguards.mdx @@ -0,0 +1,42 @@ +--- +id: collection-and-safeguards +slug: /measurement/collection-and-safeguards +title: Collection and safeguards +description: Collection rules, baseline interpretation, event contracts and privacy safeguards. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +{/* preserved:start */} +Introduction depends on the [roadmap gates](../roadmap.mdx); these pages do not describe installed controls or measured company results. + +### Decide what counts before counting + +Use the [metric definitions](./index.mdx#metric-definitions) for exact definitions. Every report states its eligible population, observation window, numerator and denominator or distribution sample count, source coverage, exclusions and calculation version. Zero denominators return “not available.” Missing telemetry is unknown, not inactivity, compliance or success. Examples are explicitly synthetic and illustrative. + +Begin with task acceptance, repository, CI, deployment and incident systems because they can independently establish outcomes. Add catalog/version inventory and task-level declarations. Client telemetry is supplementary and version-dependent; enabling rich tool details may collect sensitive commands and identifiers. Do not enable it merely to count skills. A normalized adapter may report approved skill identifiers without forwarding raw arguments. + +Validated success uses all eligible initiated tasks whose predeclared observation window has matured. Failed and abandoned tasks remain nonsuccess. Unresolved or unobserved outcomes remain in the denominator, with known-outcome coverage shown alongside the conservative confirmed-success share. They are not relabeled as observed failure. A task cancelled before start is excluded; cancellation after start remains visible. Count a task once across retries and multiple skills. For timeliness, also show unfinished work and age; accepted-only lead-time distributions have survivorship limitations. + +Track all five service-level DORA measures: commit-to-production lead time, deployment frequency, change fail rate, failed deployment recovery time and deployment rework rate. Do not substitute PR-open-to-merge time or all-incident MTTR. Pair them with matured escaped-quality rates, policy effects, total cost per accepted outcome, support coverage, adoption, human experience and catalog maintenance. Keep customer/business outcomes linked to their departmental hypotheses; do not infer conversion, trust or revenue improvement from delivery telemetry. + +### Baseline and interpretation + +Before a pilot, freeze eligibility and acceptance rules and verify event joins. Collect prospective no-skill/current-practice evidence over a complete comparable workflow cycle and until the preregistered precision/sample requirement is met. Do not select a convenient end date after seeing results. Retain task class, service, skill, client/model configuration and concurrent changes. A matched or randomized comparison may be used when practical and ethical; label observational trends as descriptive and report confounding, selection and uncertainty. + +Elapsed time and human effort are different quantities. Parallel agents may reduce waiting while increasing review or compute. Include failed attempts and human review/rework in cost, do not multiply a multi-skill task's value across every invocation, and do not convert survey perceptions into proven time savings. Small or incomplete populations yield evidence limits, not an ROI percentage. + +### Event and privacy contract + +Use a versioned event envelope and required fields per event. Source producer plus event ID deduplicates transport retries. Conflicting payloads under the same key are quarantined. Attempts have distinct IDs under one task. Domain IDs deduplicate deployments, defects and approvals even when multiple feeds describe them. Late arrivals produce versioned aggregate corrections; never silently overwrite a published result. Preserve occurrence and receipt times, reject impossible orderings and flag clock skew. Missing terminal events remain unresolved until the work owner reconciles them. + +Many-to-many task/attempt/skill/client associations preserve exact immutable versions. Unknown version stays null with a reason; never assign the latest revision retroactively. Client declarations, server facts and self-reports remain separately labeled. A skill-loaded event is evidence of loading only. The minimal prototype can operate without actor identity except where a lawful, necessary adoption denominator requires pseudonymous cohort membership. + +The privacy starting proposal is 30 days for raw structured analytics events and 12 months for aggregated trends, subject to the responsible owners' review. Keep the approved catalog/approval record for the supported revision's lifetime and its required audit period, which must be decided separately. Incident evidence uses a restricted, explicitly authorized hold and deletion schedule, not an indefinite analytics exception. Suppress reports below five people and prohibit drill-down or overlapping cohort slices that reconstruct identities. Five is a proposed minimum, not a guarantee against re-identification. No raw prompts, code, screens, customer cases or secrets are collected by default. Publish the purpose, lawful collection basis, access/deletion route and retention before collecting. Consent is not presumed to be an appropriate employment-law basis. +{/* preserved:end */} + +Return to the [Skills Measurement index](./index.mdx) for every metric definition. + +
diff --git a/docs/strategy/measurement/confirmed-policy-effects.mdx b/docs/strategy/measurement/confirmed-policy-effects.mdx new file mode 100644 index 0000000..a43356e --- /dev/null +++ b/docs/strategy/measurement/confirmed-policy-effects.mdx @@ -0,0 +1,36 @@ +--- +id: confirmed-policy-effects +slug: /measurement/confirmed-policy-effects +title: Confirmed policy effects +description: Definition, collection evidence and limitations for confirmed policy effects. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Eligible evaluated attempts with an actual confirmed prohibited effect; report attempts, blocked attempts and effects separately. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct attempt_ids with confirmed prohibited action effect | +| Denominator | All evaluated eligible attempt_ids including failed and abandoned runs | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Governed skill trials or supported workflow attempts, reported separately for eval and live populations. | +| Exclusions | Out-of-scope runs; detection gaps remain coverage issues, not exclusions from known initiated population. | +| Events and source | attempt.started; policy.violation; gate.blocked; security audit and incident triage | +| Owner / cadence | Architecture & Security / Immediate incident action; weekly aggregates | +| Missing evidence | Unknown outcomes remain visible; zero detected effects never proves safety. Missing audits make rate provisional. | +| Gaming and limits | Hide incidents or inflate harmless trials; absolute severity counts and fixed task mix accompany rate. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Blocked attempts are not actual effects; any critical effect blocks promotion regardless of aggregate rate. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/cost-per-accepted-outcome.mdx b/docs/strategy/measurement/cost-per-accepted-outcome.mdx new file mode 100644 index 0000000..dd3f388 --- /dev/null +++ b/docs/strategy/measurement/cost-per-accepted-outcome.mdx @@ -0,0 +1,36 @@ +--- +id: cost-per-accepted-outcome +slug: /measurement/cost-per-accepted-outcome +title: Cost per accepted outcome +description: Definition, collection evidence and limitations for cost per accepted outcome. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Allocated total workflow cost including unsuccessful attempts, licenses, compute, tooling and human review/rework divided by accepted outcomes. + +| Contract | Definition | +| --- | --- | +| Numerator | Sum allocated costs in one approved reporting currency for all tasks/attempts in matured start cohort | +| Denominator | Distinct independently accepted tasks in same cohort | +| Expression | N / D | +| Aggregation | ratio | +| Unit | currency/accepted outcome | +| Eligible population | Matured eligible started-task cohort, with all attempt costs through as_of and allocated shared period costs. | +| Exclusions | Unrelated projects and documented taxes/transfers outside chosen accounting basis; never failed/abandoned attempts. | +| Events and source | cost.recorded; task.accepted; finance invoices, usage ledger, sampled effort with approved rates | +| Owner / cadence | Finance & Accounting Squad / Monthly with late invoice restatement | +| Missing evidence | Missing cost buckets produce partial estimate plus coverage, not a total. Mixed currency requires dated approved conversion; zero accepted outcomes = null with full cost shown. | +| Gaming and limits | Count tokens only or hide labor/failures; reconcile invoice and allocation totals without double counting. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Allocation and sampled labor carry uncertainty; no ROI without independently measured business benefit. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/deployment-frequency.mdx b/docs/strategy/measurement/deployment-frequency.mdx new file mode 100644 index 0000000..4841382 --- /dev/null +++ b/docs/strategy/measurement/deployment-frequency.mdx @@ -0,0 +1,36 @@ +--- +id: deployment-frequency +slug: /measurement/deployment-frequency +title: Deployment frequency +description: Definition, collection evidence and limitations for deployment frequency. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Unimplemented metric — no live telemetry.** This is a prospective contract, not an implemented calculation or collector. + +{/* contract:start */} +Distinct successful logical production deployments per complete service-day of observation. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct service_id+deployment_id successful production deployments in window | +| Denominator | Observed service-days in window | +| Expression | N / D | +| Aggregation | ratio | +| Unit | deployments/service-day | +| Eligible population | Declared production services and complete UTC observation days; report each service before pooled views. | +| Exclusions | Test/staging, failed attempts without production effect, duplicate callbacks and retries of the same rollout. | +| Events and source | deployment.completed; authoritative deployment service | +| Owner / cadence | Platform & IT / Weekly per service | +| Missing evidence | Missing feed periods not zero; report coverage and no unqualified overall rate when period incomplete. | +| Gaming and limits | Split deployments without value; pair with success, rework and failure measures. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. More deployments alone do not establish better performance; affected unsuccessful changes remain in change-fail population. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/deployment-rework-rate.mdx b/docs/strategy/measurement/deployment-rework-rate.mdx new file mode 100644 index 0000000..ecb663e --- /dev/null +++ b/docs/strategy/measurement/deployment-rework-rate.mdx @@ -0,0 +1,36 @@ +--- +id: deployment-rework-rate +slug: /measurement/deployment-rework-rate +title: Deployment rework rate +description: Definition, collection evidence and limitations for deployment rework rate. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Unplanned production deployment effects triggered by a production incident as a share of deployment effects. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct incident-linked deployment_ids classified unplanned remediation | +| Denominator | All distinct production deployment_ids with production effect in window | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Production deployment effects in window with immutable planned/unplanned classification and incident linkage. | +| Exclusions | Nonproduction, no-effect retries and duplicates; do not exclude unclassified changes silently. | +| Events and source | deployment.completed; incident.linked; release intent classification | +| Owner / cadence | Platform & IT / Weekly | +| Missing evidence | Unknown classification stays denominator; observed numerator is lower bound until reconciliation; show classification coverage. | +| Gaming and limits | Call hotfixes planned; freeze reason and independently reconcile incident/deployment links. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Distinct from failure rate: one failed deployment can lead to multiple separate rework deployments. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/failed-deployment-recovery-time.mdx b/docs/strategy/measurement/failed-deployment-recovery-time.mdx new file mode 100644 index 0000000..6c17e54 --- /dev/null +++ b/docs/strategy/measurement/failed-deployment-recovery-time.mdx @@ -0,0 +1,36 @@ +--- +id: failed-deployment-recovery-time +slug: /measurement/failed-deployment-recovery-time +title: Failed deployment recovery time +description: Definition, collection evidence and limitations for failed deployment recovery time. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Unimplemented metric — no live telemetry.** This is a prospective contract, not an implemented calculation or collector. + +{/* contract:start */} +Mean elapsed time from production deployment failure onset to service restoration, with median/p90 and unresolved count/age. + +| Contract | Definition | +| --- | --- | +| Numerator | Sum of recovery_at minus failure_started_at in hours for recovered eligible deployment failures | +| Denominator | Count of recovered eligible deployment failures with valid timestamps | +| Expression | N / D | +| Aggregation | mean | +| Unit | hours/failure | +| Eligible population | Production deployment failures beginning in window; latest verified restoration if a recovery is rescinded. | +| Exclusions | Nondeployment incidents; invalid timestamp pairs reported missing, not treated as instantaneous recovery. | +| Events and source | deployment.failed; service.restored; incident service | +| Owner / cadence | Platform & IT / Weekly with unresolved failures | +| Missing evidence | Unresolved failures reported separately and excluded only from recovered-duration distribution; prevent declaring improvement without their visibility. | +| Gaming and limits | Premature recovery declarations or omit hard cases; require operational readback and carry unresolved ages. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Not all-incident MTTR; recovered-only sample has survivorship bias. p90 nearest rank. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement.mdx b/docs/strategy/measurement/index.mdx similarity index 56% rename from docs/strategy/measurement.mdx rename to docs/strategy/measurement/index.mdx index 81f3f0e..6bafc64 100644 --- a/docs/strategy/measurement.mdx +++ b/docs/strategy/measurement/index.mdx @@ -1,5 +1,6 @@ --- -id: measurement +id: index +slug: /measurement title: Skills Measurement sidebar_label: Measurement description: Which KPIs to collect during skills adoption, where the evidence comes from, how to calculate them and when to report. @@ -28,44 +29,71 @@ Start with these four questions. Capture inputs as work happens; do not reconstr ### Are tasks reaching acceptance? -**Collect:** task starts and independent acceptance, failure or abandonment records from the work tracker. **Calculate:** accepted tasks / all eligible starts whose observation period has ended × 100. Failed, abandoned and unresolved tasks stay in the denominator; show unknown outcomes and coverage separately. This is [validated task success](./reference/measurement-contracts.mdx#validated-task-success), not a count of agent completions. **Weekly:** Quality Engineering reviews the pilot cohort. +**Collect:** task starts and independent acceptance, failure or abandonment records from the work tracker. **Calculate:** accepted tasks / all eligible starts whose observation period has ended × 100. Failed, abandoned and unresolved tasks stay in the denominator; show unknown outcomes and coverage separately. This is [validated task success](./validated-task-success.mdx), not a count of agent completions. **Weekly:** Quality Engineering reviews the pilot cohort. ### Is quality holding up? -**Collect:** accepted change IDs, linked confirmed defects, severity and follow-up dates from review and defect records; retain PR/CI evidence. **Calculate:** changes with at least one confirmed defect / accepted changes with completed follow-up × 100. Count each affected change once, not each defect. Missing links do not mean clean work. **Monthly:** Quality Engineering reviews [escaped quality](./reference/measurement-contracts.mdx#changes-with-escaped-quality-issues) after follow-up, alongside severe incident counts. +**Collect:** accepted change IDs, linked confirmed defects, severity and follow-up dates from review and defect records; retain PR/CI evidence. **Calculate:** changes with at least one confirmed defect / accepted changes with completed follow-up × 100. Count each affected change once, not each defect. Missing links do not mean clean work. **Monthly:** Quality Engineering reviews [escaped quality](./changes-with-escaped-quality-issues.mdx) after follow-up, alongside severe incident counts. ### What does an accepted outcome cost? -**Collect:** invoices and usage costs plus sampled human review/rework minutes at approved rates, including unsuccessful attempts and allocated licenses/tooling. **Calculate:** total allocated workflow cost / independently accepted tasks in the same matured start cohort. Keep effort separate from waiting time. Missing buckets make this a partial estimate, not total cost. **Monthly:** Finance & Accounting reconciles [cost per accepted outcome](./reference/measurement-contracts.mdx#cost-per-accepted-outcome), restating late invoices. +**Collect:** invoices and usage costs plus sampled human review/rework minutes at approved rates, including unsuccessful attempts and allocated licenses/tooling. **Calculate:** total allocated workflow cost / independently accepted tasks in the same matured start cohort. Keep effort separate from waiting time. Missing buckets make this a partial estimate, not total cost. **Monthly:** Finance & Accounting reconciles [cost per accepted outcome](./cost-per-accepted-outcome.mdx), restating late invoices. ### Can people confidently verify the work? -**Collect:** an anonymous 1–5 survey after a pilot workflow cycle: verification confidence, manageable workload and usefulness as separate items. **Calculate:** sum of valid confidence scores / valid responses; also show the distribution and responses / invitations × 100. This guides enablement, not claims of saved time. **After each pilot cycle:** People & Administration reviews [confidence and cognitive load](./reference/measurement-contracts.mdx#verification-confidence-and-cognitive-load); protect small groups. +**Collect:** an anonymous 1–5 survey after a pilot workflow cycle: verification confidence, manageable workload and usefulness as separate items. **Calculate:** sum of valid confidence scores / valid responses; also show the distribution and responses / invitations × 100. This guides enablement, not claims of saved time. **After each pilot cycle:** People & Administration reviews [confidence and cognitive load](./verification-confidence-and-cognitive-load.mdx); protect small groups. ### What a pilot report looks like **Illustrative arithmetic, not company results or targets.** The bundled synthetic fixture shows four matured starts: one accepted, one failed, one abandoned and one unresolved. Confirmed success is **1 / 4 × 100 = 25%**, with **75%** known-outcome coverage. A newer start has not finished its observation period and is shown separately. -Recorded compute spend is **EUR 60 / 1 = EUR 60** per accepted outcome—a **partial estimate** because licenses, tooling, review and rework are missing. Five survey responses total 20 points: **20 / 5 = 4** out of 5; response rate is **5 / 10 × 100 = 50%**. The report calls for reconciling the unknown task and missing costs, not claiming a skills benefit. Reproduce it with `node scripts/ai-kpi.mjs`; [fixture and limits](./reference/measurement-contracts.mdx#run-the-synthetic-measurement-example). +Recorded compute spend is **EUR 60 / 1 = EUR 60** per accepted outcome—a **partial estimate** because licenses, tooling, review and rework are missing. Five survey responses total 20 points: **20 / 5 = 4** out of 5; response rate is **5 / 10 × 100 = 50%**. The report calls for reconciling the unknown task and missing costs, not claiming a skills benefit. Reproduce it with `node scripts/ai-kpi.mjs`; [fixture and limits](./synthetic-example.mdx). ## 3. Extend collection as adoption grows Keep those outcome measures, then add reach. An adoption percentage is meaningless without a known eligible population. -**Are supported skills available?** Freeze the expected active installation inventory; compare observed bundle/adapter revisions with the approved support and revocation records. [Supported bundle coverage](./reference/measurement-contracts.mdx#supported-bundle-coverage) = fresh, verified supported installations / all expected active installations × 100. Offline or stale entries stay in the denominator. **Weekly and at revocation:** Platform & IT reconciles coverage. +**Are supported skills available?** Freeze the expected active installation inventory; compare observed bundle/adapter revisions with the approved support and revocation records. [Supported bundle coverage](./supported-bundle-coverage.mdx) = fresh, verified supported installations / all expected active installations × 100. Offline or stale entries stay in the denominator. **Weekly and at revocation:** Platform & IT reconciles coverage. -**Are approved workflows being used?** Freeze the lawfully enrolled participant cohort; collect workflow-linked use of an approved revision through a qualified adapter or a separately labeled declaration. [Approved-workflow adoption](./reference/measurement-contracts.mdx#approved-workflow-adoption) = eligible participants with at least one observed approved use / eligible invited participants × 100. Report this as an observed lower bound with source coverage: missing use is unknown. Installation and loading are not use. **Monthly:** Data & AI reviews reach without invocation quotas or employee rankings. +**Are approved workflows being used?** Freeze the lawfully enrolled participant cohort; collect workflow-linked use of an approved revision through a qualified adapter or a separately labeled declaration. [Approved-workflow adoption](./approved-workflow-adoption.mdx) = eligible participants with at least one observed approved use / eligible invited participants × 100. Report this as an observed lower bound with source coverage: missing use is unknown. Installation and loading are not use. **Monthly:** Data & AI reviews reach without invocation quotas or employee rankings. ## 4. Keep the collection running -**Are skills still maintained?** From the versioned catalog, join each active approved revision to its owner, substantive review evidence and due date. [Review coverage](./reference/measurement-contracts.mdx#skills-within-substantive-review-window) = revisions with all three current / all active approved revisions × 100. **Monthly and each release:** Data & AI reports overdue/ownerless revisions and deprecated or revoked versions still in use—not just a percentage. +**Are skills still maintained?** From the versioned catalog, join each active approved revision to its owner, substantive review evidence and due date. [Review coverage](./skills-within-substantive-review-window.mdx) = revisions with all three current / all active approved revisions × 100. **Monthly and each release:** Data & AI reports overdue/ownerless revisions and deprecated or revoked versions still in use—not just a percentage. At each report, compare like workflows with the baseline and publish the period, counts, exclusions and source coverage. Zero denominators mean “not available.” Trends are descriptive, not causal ROI. Act on critical policy effects immediately; add selection precision/recall at qualification and service-level delivery or departmental outcome measures when their source links are ready. Agree purpose, lawful basis, access and retention first. No raw prompts, code or secrets by default; suppress people cohorts below five and prevent identifying drill-downs. -[Full KPI reference →](./reference/measurement-contracts.mdx) preserves all definitions, collection safeguards and the distinction between synthetic calculations and unimplemented metrics. +## Metric definitions -**Next:** [Maintain skills using the evidence →](./skills-maintenance.mdx) +There are **16 contracts: 9 prototype calculations and 7 unimplemented metrics**. The prototype has no network collectors. None of these counts establishes live telemetry. + +The nine calculations cover task success, selection precision and recall, change fail rate, deployment rework, escaped quality, policy effects, cost and the primary human-experience item. The seven omissions are supported bundle coverage, approved-workflow adoption, change lead time, deployment frequency, failed deployment recovery time, substantive-review coverage and workflow-specific business acceptance. + +Each contract is a prospective collection design. No live source or observed baseline is established. N and D refer to the numerator and denominator shown for that metric. + +Selection measures concern qualification; coverage and adoption concern reach; DORA measures concern service delivery. Outcome, safety, cost, experience and maintenance measures retain their distinct populations. + +1. [Validated task success](./validated-task-success.mdx) — prototype calculation +2. [Skill selection precision](./skill-selection-precision.mdx) — prototype calculation +3. [Skill selection recall](./skill-selection-recall.mdx) — prototype calculation +4. [Supported bundle coverage](./supported-bundle-coverage.mdx) — unimplemented metric +5. [Approved-workflow adoption](./approved-workflow-adoption.mdx) — unimplemented metric +6. [Change lead time](./change-lead-time.mdx) — unimplemented metric +7. [Deployment frequency](./deployment-frequency.mdx) — unimplemented metric +8. [Change fail rate](./change-fail-rate.mdx) — prototype calculation +9. [Failed deployment recovery time](./failed-deployment-recovery-time.mdx) — unimplemented metric +10. [Deployment rework rate](./deployment-rework-rate.mdx) — prototype calculation +11. [Changes with escaped quality issues](./changes-with-escaped-quality-issues.mdx) — prototype calculation +12. [Confirmed policy effects](./confirmed-policy-effects.mdx) — prototype calculation +13. [Cost per accepted outcome](./cost-per-accepted-outcome.mdx) — prototype calculation +14. [Verification confidence and cognitive load](./verification-confidence-and-cognitive-load.mdx) — prototype calculation +15. [Skills within substantive review window](./skills-within-substantive-review-window.mdx) — unimplemented metric +16. [Workflow-specific business acceptance](./workflow-specific-business-acceptance.mdx) — unimplemented metric + +Read [Collection and safeguards](./collection-and-safeguards.mdx) for shared baseline, event and privacy rules. [Run the synthetic measurement example](./synthetic-example.mdx) for reproducible arithmetic and limits. + +**Next:** [Maintain skills using the evidence →](../skills-maintenance.mdx) diff --git a/docs/strategy/measurement/skill-selection-precision.mdx b/docs/strategy/measurement/skill-selection-precision.mdx new file mode 100644 index 0000000..0b06042 --- /dev/null +++ b/docs/strategy/measurement/skill-selection-precision.mdx @@ -0,0 +1,36 @@ +--- +id: skill-selection-precision +slug: /measurement/skill-selection-precision +title: Skill selection precision +description: Definition, collection evidence and limitations for skill selection precision. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Correct activations among activations on an independently labeled evaluation corpus. + +| Contract | Definition | +| --- | --- | +| Numerator | True-positive activations for the requested workflow | +| Denominator | True-positive plus false-positive activations | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | All candidate/client labeled evaluation task-trials with known activation and relevance. | +| Exclusions | Unlabeled/unknown activations excluded with coverage; no cherry-picked failed trials. | +| Events and source | eval.selection; versioned labeled corpus and runner | +| Owner / cadence | Data & AI / Each skill/client qualification | +| Missing evidence | Unknown activation/relevance is unscorable; publish missing count and deny qualification if planned coverage incomplete. | +| Gaming and limits | Choose easy labels or hide negative cases; freeze corpus and independently review labels. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Does not show execution success; precision depends on corpus prevalence. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/skill-selection-recall.mdx b/docs/strategy/measurement/skill-selection-recall.mdx new file mode 100644 index 0000000..4d49eca --- /dev/null +++ b/docs/strategy/measurement/skill-selection-recall.mdx @@ -0,0 +1,36 @@ +--- +id: skill-selection-recall +slug: /measurement/skill-selection-recall +title: Skill selection recall +description: Definition, collection evidence and limitations for skill selection recall. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Relevant task-trials in which the intended skill activates. + +| Contract | Definition | +| --- | --- | +| Numerator | True-positive activations | +| Denominator | True-positive plus false-negative task-trials | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Independently labeled relevant evaluation task-trials for a skill/client tuple. | +| Exclusions | Irrelevant tasks; missing relevance or activation reported as unscorable, not negative. | +| Events and source | eval.selection; versioned labeled corpus and runner | +| Owner / cadence | Data & AI / Each skill/client qualification | +| Missing evidence | Report evaluation coverage; null if no relevant labeled tasks. | +| Gaming and limits | Overtrigger to maximize recall; always pair with precision and negative cases. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Loading the right skill is not adherence. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/skills-within-substantive-review-window.mdx b/docs/strategy/measurement/skills-within-substantive-review-window.mdx new file mode 100644 index 0000000..bc395e3 --- /dev/null +++ b/docs/strategy/measurement/skills-within-substantive-review-window.mdx @@ -0,0 +1,36 @@ +--- +id: skills-within-substantive-review-window +slug: /measurement/skills-within-substantive-review-window +title: Skills within substantive review window +description: Definition, collection evidence and limitations for skills within substantive review window. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Unimplemented metric — no live telemetry.** This is a prospective contract, not an implemented calculation or collector. + +{/* contract:start */} +Active approved skill revisions with an accountable owner and completed substantive review not past due. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct active approved skill revisions with current owner, review evidence and review_due_at >= as_of | +| Denominator | All active approved skill revisions in catalog snapshot | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Approved revisions currently supported for use at as_of. | +| Exclusions | Experimental and retired revisions; deprecated/revoked in-use counts reported independently, not hidden. | +| Events and source | catalog.published; catalog.reviewed; catalog.deprecated; catalog.revoked; version inventory | +| Owner / cadence | Data & AI / Monthly and each release | +| Missing evidence | Missing owner/date/evidence outside numerator; show ownerless, deprecated-in-use and revoked-in-use absolute counts. | +| Gaming and limits | Rubber-stamp review dates; require linked regression/security/usefulness evidence. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. A current review is not proof of effectiveness; consumer inventory may miss shadow copies. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/supported-bundle-coverage.mdx b/docs/strategy/measurement/supported-bundle-coverage.mdx new file mode 100644 index 0000000..50d93e5 --- /dev/null +++ b/docs/strategy/measurement/supported-bundle-coverage.mdx @@ -0,0 +1,36 @@ +--- +id: supported-bundle-coverage +slug: /measurement/supported-bundle-coverage +title: Supported bundle coverage +description: Definition, collection evidence and limitations for supported bundle coverage. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Unimplemented metric — no live telemetry.** This is a prospective contract, not an implemented calculation or collector. + +{/* contract:start */} +Inventoried active installations with verified effective approved, supported and nonrevoked bundle/adapter tuple. + +| Contract | Definition | +| --- | --- | +| Numerator | Eligible installation_ids with valid inventory evidence within approved freshness window and matching support matrix | +| Denominator | All expected active installation_ids in the managed pilot inventory | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Active installation inventory frozen for report, including invisible/offline entries until authoritative offboarding. | +| Exclusions | Verified decommissioned installations and nonpilot assets; lack of telemetry alone never excludes. | +| Events and source | inventory.observed; catalog.published; catalog.revoked; device inventory | +| Owner / cadence | Platform & IT / Weekly and at revocation | +| Missing evidence | Unknown/stale versions count outside numerator, inside denominator. Show inventory discovery coverage versus eligible device roster; if roster unknown metric is provisional. | +| Gaming and limits | Remove silent devices or trust declared versions; reconcile managed inventory and separately label attestation trust. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Even verified file digests do not prove a private model followed the content. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/synthetic-example.mdx b/docs/strategy/measurement/synthetic-example.mdx new file mode 100644 index 0000000..15c8335 --- /dev/null +++ b/docs/strategy/measurement/synthetic-example.mdx @@ -0,0 +1,22 @@ +--- +id: synthetic-example +slug: /measurement/synthetic-example +title: Run the synthetic measurement example +description: Reproduce the synthetic KPI calculations and understand their limits. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +{/* preserved:start */} +**Executed synthetic example — not Eldorado results.** The bundled fixture produces 1 independently accepted task from 4 matured starts: 25% confirmed success. One unresolved task remains in that denominator; known-outcome coverage is 75%. Selection precision and recall are each 1 / 2 = 50%. Recorded synthetic compute cost is EUR 60 / 1 accepted outcome = EUR 60 per accepted outcome, a partial estimate: licenses, tooling, review and rework are missing. These deliberately small samples demonstrate arithmetic and missingness, not a baseline, target or ROI claim. + +From the private documentation checkout, run `node scripts/ai-kpi.mjs` and `node --test scripts/ai-kpi.test.mjs`. The fixture `scripts/fixtures/ai-kpi-synthetic.json` is labeled synthetic; every result carries the same label and calculation version. This prototype has no network collectors and does not enable employee telemetry. + +The executable covers validated success, selection precision/recall, change fail and rework rates, escaped quality, policy effects, cost and human experience. Other catalog metrics remain contracts rather than implemented collectors; output lists the omissions. Snapshot inputs are simplified, pre-reconciled records, not a production ingestion system. Conflicting duplicates reject the calculation rather than silently choosing a value. +{/* preserved:end */} + +Return to the [Skills Measurement index](./index.mdx) for every metric definition. + +
diff --git a/docs/strategy/measurement/validated-task-success.mdx b/docs/strategy/measurement/validated-task-success.mdx new file mode 100644 index 0000000..a309933 --- /dev/null +++ b/docs/strategy/measurement/validated-task-success.mdx @@ -0,0 +1,36 @@ +--- +id: validated-task-success +slug: /measurement/validated-task-success +title: Validated task success +description: Definition, collection evidence and limitations for validated task success. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Confirmed independently accepted tasks as a share of all eligible initiated tasks in a matured start cohort; a conservative confirmed-success share, not observed failure rate. + +| Contract | Definition | +| --- | --- | +| Numerator | Distinct matured task_ids with valid independent task.accepted evidence by as_of | +| Denominator | All distinct eligible task_ids started in window with started_at + preregistered task_observation_hours <= as_of | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | Tasks in a preregistered workflow cohort, started in window; each counted once across retries, skills and clients. | +| Exclusions | Cancelled before start, sandbox/evaluation tasks and out-of-scope workflows; never post-start failures/abandonments. | +| Events and source | task.started; task.accepted; task.failed; task.abandoned; task.cancelled; work-management acceptance record | +| Owner / cadence | Quality Engineering / Weekly pilot; monthly departmental outcome review | +| Missing evidence | Unresolved/missing terminal outcomes remain in denominator with unknown count and known-outcome coverage; do not label them failed. Invalid/missing start excluded from calculation but flagged as orphan evidence. | +| Gaming and limits | Split easy tasks or omit starts; reconcile starts to approved backlog and audit acceptance rubrics. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Maturation horizon is workflow-specific and must be approved; no automatic abandonment on timeout. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/verification-confidence-and-cognitive-load.mdx b/docs/strategy/measurement/verification-confidence-and-cognitive-load.mdx new file mode 100644 index 0000000..6c306b0 --- /dev/null +++ b/docs/strategy/measurement/verification-confidence-and-cognitive-load.mdx @@ -0,0 +1,36 @@ +--- +id: verification-confidence-and-cognitive-load +slug: /measurement/verification-confidence-and-cognitive-load +title: Verification confidence and cognitive load +description: Definition, collection evidence and limitations for verification confidence and cognitive load. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Prototype calculation — no live telemetry.** The synthetic example calculates this metric; it has no network collectors. + +{/* contract:start */} +Three separate anonymous 1–5 item distributions: confidence in verifying output, manageable workload, usefulness. Primary item: verification confidence mean; report full distributions and response rate. + +| Contract | Definition | +| --- | --- | +| Numerator | Sum valid 1–5 verification-confidence responses, one per participant per survey wave | +| Denominator | Count valid responses to that item | +| Expression | N / D | +| Aggregation | mean | +| Unit | points (1–5) | +| Eligible population | Invited eligible cohort at fixed survey wave; voluntary responses. | +| Exclusions | Invalid/out-of-range answers and superseded duplicate submission; nonresponders stay in response-rate denominator. | +| Events and source | survey.response; anonymous survey with single-use token and aggregate invitation count | +| Owner / cadence | People & Administration Squad / After pilot workflow cycle; periodic follow-up | +| Missing evidence | Report valid response count / invited count and item nonresponse; no imputation. Suppress people cohorts below five. | +| Gaming and limits | Pressure to respond positively; no manager access to individual responses, avoid identifying free text. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Perception and selection bias; not measured time savings. No composite productivity index. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/measurement/workflow-specific-business-acceptance.mdx b/docs/strategy/measurement/workflow-specific-business-acceptance.mdx new file mode 100644 index 0000000..adde781 --- /dev/null +++ b/docs/strategy/measurement/workflow-specific-business-acceptance.mdx @@ -0,0 +1,36 @@ +--- +id: workflow-specific-business-acceptance +slug: /measurement/workflow-specific-business-acceptance +title: Workflow-specific business acceptance +description: Definition, collection evidence and limitations for workflow-specific business acceptance. +--- + +
+ +**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. + +**Unimplemented metric — no live telemetry.** This is a prospective contract, not an implemented calculation or collector. + +{/* contract:start */} +Independently accepted domain outcomes against a predeclared business rubric, separate from technical completion. + +| Contract | Definition | +| --- | --- | +| Numerator | Eligible completed-observation task_ids meeting the department-specific outcome rubric | +| Denominator | All eligible initiated workflow task_ids with matured business-outcome horizon | +| Expression | N / D * 100 | +| Aggregation | percent | +| Unit | % | +| Eligible population | One named departmental workflow and predeclared start cohort; never pool unlike rubrics. | +| Exclusions | Cancelled-before-start and unrelated workflows; unsuccessful or abandoned eligible work remains. | +| Events and source | task.started; business.accepted; departmental case/content/reconciliation/onboarding review record | +| Owner / cadence | Product Management / Each pilot cycle with owning department | +| Missing evidence | Unknown follow-up stays denominator with unresolved count; no claim actual business outcome changed without source evidence. | +| Gaming and limits | Weaken rubric or count drafts as customer benefit; lock rubric and independently audit acceptance. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Domain owner signs acceptance: Operations service accuracy; Marketing claims/rights; Finance reconciliation accuracy; HR policy correctness; Product usability/value. Not revenue uplift. | +{/* contract:end */} + +Read the shared [collection and safeguards](./collection-and-safeguards.mdx) before collecting data; see the [synthetic example and limits](./synthetic-example.mdx). + +Return to [Skills Measurement](./index.mdx). + +
diff --git a/docs/strategy/reference/measurement-contracts.mdx b/docs/strategy/reference/measurement-contracts.mdx deleted file mode 100644 index aea029e..0000000 --- a/docs/strategy/reference/measurement-contracts.mdx +++ /dev/null @@ -1,363 +0,0 @@ ---- -id: measurement-contracts -title: Skills KPI reference -description: Full KPI contracts, minimal events, illustrative calculations, privacy and limits. ---- - -
- -Return to the [Skills Measurement collection plan](../measurement.mdx). - -**Target operating model — recommended contracts, not installed collection.** All owner and cadence entries below are proposals to confirm before the pilot, not assigned responsibilities or approved schedules. - -There are **16 contracts: 9 prototype calculations and 7 unimplemented metrics**. The prototype has no network collectors. None of these counts establishes live telemetry. - -The nine calculations cover task success, selection precision and recall, change fail rate, deployment rework, escaped quality, policy effects, cost and the primary human-experience item. The seven omissions are supported bundle coverage, approved-workflow adoption, change lead time, deployment frequency, failed deployment recovery time, substantive-review coverage and workflow-specific business acceptance. - -## Collection and safeguards - -Introduction depends on the [roadmap gates](../roadmap.mdx); these pages do not describe installed controls or measured company results. - -### Decide what counts before counting - -Use the metric contracts below for exact definitions. Every report states its eligible population, observation window, numerator and denominator or distribution sample count, source coverage, exclusions and calculation version. Zero denominators return “not available.” Missing telemetry is unknown, not inactivity, compliance or success. Examples are explicitly synthetic and illustrative. - -Begin with task acceptance, repository, CI, deployment and incident systems because they can independently establish outcomes. Add catalog/version inventory and task-level declarations. Client telemetry is supplementary and version-dependent; enabling rich tool details may collect sensitive commands and identifiers. Do not enable it merely to count skills. A normalized adapter may report approved skill identifiers without forwarding raw arguments. - -Validated success uses all eligible initiated tasks whose predeclared observation window has matured. Failed and abandoned tasks remain nonsuccess. Unresolved or unobserved outcomes remain in the denominator, with known-outcome coverage shown alongside the conservative confirmed-success share. They are not relabeled as observed failure. A task cancelled before start is excluded; cancellation after start remains visible. Count a task once across retries and multiple skills. For timeliness, also show unfinished work and age; accepted-only lead-time distributions have survivorship limitations. - -Track all five service-level DORA measures: commit-to-production lead time, deployment frequency, change fail rate, failed deployment recovery time and deployment rework rate. Do not substitute PR-open-to-merge time or all-incident MTTR. Pair them with matured escaped-quality rates, policy effects, total cost per accepted outcome, support coverage, adoption, human experience and catalog maintenance. Keep customer/business outcomes linked to their departmental hypotheses; do not infer conversion, trust or revenue improvement from delivery telemetry. - -### Baseline and interpretation - -Before a pilot, freeze eligibility and acceptance rules and verify event joins. Collect prospective no-skill/current-practice evidence over a complete comparable workflow cycle and until the preregistered precision/sample requirement is met. Do not select a convenient end date after seeing results. Retain task class, service, skill, client/model configuration and concurrent changes. A matched or randomized comparison may be used when practical and ethical; label observational trends as descriptive and report confounding, selection and uncertainty. - -Elapsed time and human effort are different quantities. Parallel agents may reduce waiting while increasing review or compute. Include failed attempts and human review/rework in cost, do not multiply a multi-skill task's value across every invocation, and do not convert survey perceptions into proven time savings. Small or incomplete populations yield evidence limits, not an ROI percentage. - -### Event and privacy contract - -Use a versioned event envelope and required fields per event. Source producer plus event ID deduplicates transport retries. Conflicting payloads under the same key are quarantined. Attempts have distinct IDs under one task. Domain IDs deduplicate deployments, defects and approvals even when multiple feeds describe them. Late arrivals produce versioned aggregate corrections; never silently overwrite a published result. Preserve occurrence and receipt times, reject impossible orderings and flag clock skew. Missing terminal events remain unresolved until the work owner reconciles them. - -Many-to-many task/attempt/skill/client associations preserve exact immutable versions. Unknown version stays null with a reason; never assign the latest revision retroactively. Client declarations, server facts and self-reports remain separately labeled. A skill-loaded event is evidence of loading only. The minimal prototype can operate without actor identity except where a lawful, necessary adoption denominator requires pseudonymous cohort membership. - -The privacy starting proposal is 30 days for raw structured analytics events and 12 months for aggregated trends, subject to the responsible owners' review. Keep the approved catalog/approval record for the supported revision's lifetime and its required audit period, which must be decided separately. Incident evidence uses a restricted, explicitly authorized hold and deletion schedule, not an indefinite analytics exception. Suppress reports below five people and prohibit drill-down or overlapping cohort slices that reconstruct identities. Five is a proposed minimum, not a guarantee against re-identification. No raw prompts, code, screens, customer cases or secrets are collected by default. Publish the purpose, lawful collection basis, access/deletion route and retention before collecting. Consent is not presumed to be an appropriate employment-law basis. - -## Metric contracts - -Each contract is a prospective collection design. No live source or observed baseline is established. N and D refer to the numerator and denominator shown for that metric. - -### Validated task success - -Confirmed independently accepted tasks as a share of all eligible initiated tasks in a matured start cohort; a conservative confirmed-success share, not observed failure rate. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct matured task_ids with valid independent task.accepted evidence by as_of | -| Denominator | All distinct eligible task_ids started in window with started_at + preregistered task_observation_hours <= as_of | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Tasks in a preregistered workflow cohort, started in window; each counted once across retries, skills and clients. | -| Exclusions | Cancelled before start, sandbox/evaluation tasks and out-of-scope workflows; never post-start failures/abandonments. | -| Events and source | task.started; task.accepted; task.failed; task.abandoned; task.cancelled; work-management acceptance record | -| Owner / cadence | Quality Engineering / Weekly pilot; monthly departmental outcome review | -| Missing evidence | Unresolved/missing terminal outcomes remain in denominator with unknown count and known-outcome coverage; do not label them failed. Invalid/missing start excluded from calculation but flagged as orphan evidence. | -| Gaming and limits | Split easy tasks or omit starts; reconcile starts to approved backlog and audit acceptance rubrics. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Maturation horizon is workflow-specific and must be approved; no automatic abandonment on timeout. | - - -### Skill selection precision - -Correct activations among activations on an independently labeled evaluation corpus. - -| Contract | Definition | -| --- | --- | -| Numerator | True-positive activations for the requested workflow | -| Denominator | True-positive plus false-positive activations | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | All candidate/client labeled evaluation task-trials with known activation and relevance. | -| Exclusions | Unlabeled/unknown activations excluded with coverage; no cherry-picked failed trials. | -| Events and source | eval.selection; versioned labeled corpus and runner | -| Owner / cadence | Data & AI / Each skill/client qualification | -| Missing evidence | Unknown activation/relevance is unscorable; publish missing count and deny qualification if planned coverage incomplete. | -| Gaming and limits | Choose easy labels or hide negative cases; freeze corpus and independently review labels. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Does not show execution success; precision depends on corpus prevalence. | - - -### Skill selection recall - -Relevant task-trials in which the intended skill activates. - -| Contract | Definition | -| --- | --- | -| Numerator | True-positive activations | -| Denominator | True-positive plus false-negative task-trials | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Independently labeled relevant evaluation task-trials for a skill/client tuple. | -| Exclusions | Irrelevant tasks; missing relevance or activation reported as unscorable, not negative. | -| Events and source | eval.selection; versioned labeled corpus and runner | -| Owner / cadence | Data & AI / Each skill/client qualification | -| Missing evidence | Report evaluation coverage; null if no relevant labeled tasks. | -| Gaming and limits | Overtrigger to maximize recall; always pair with precision and negative cases. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Loading the right skill is not adherence. | - - -### Supported bundle coverage - -Inventoried active installations with verified effective approved, supported and nonrevoked bundle/adapter tuple. - -| Contract | Definition | -| --- | --- | -| Numerator | Eligible installation_ids with valid inventory evidence within approved freshness window and matching support matrix | -| Denominator | All expected active installation_ids in the managed pilot inventory | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Active installation inventory frozen for report, including invisible/offline entries until authoritative offboarding. | -| Exclusions | Verified decommissioned installations and nonpilot assets; lack of telemetry alone never excludes. | -| Events and source | inventory.observed; catalog.published; catalog.revoked; device inventory | -| Owner / cadence | Platform & IT / Weekly and at revocation | -| Missing evidence | Unknown/stale versions count outside numerator, inside denominator. Show inventory discovery coverage versus eligible device roster; if roster unknown metric is provisional. | -| Gaming and limits | Remove silent devices or trust declared versions; reconcile managed inventory and separately label attestation trust. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Even verified file digests do not prove a private model followed the content. | - - -### Approved-workflow adoption - -Eligible participants with at least one approved-skill workflow use; usefulness assessed separately through accepted outcomes and feedback. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct eligible pseudonymous participants with workflow-linked skill.used on approved revision | -| Denominator | Distinct eligible participants in frozen invited cohort | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Consented or otherwise lawfully enrolled pilot cohort for the period; supported workflow available. | -| Exclusions | Outside pilot and formally recorded leave before period; nonresponders remain eligible, not excluded. | -| Events and source | skill.used; cohort.snapshot; optional qualified client adapter or declaration | -| Owner / cadence | Data & AI / Monthly | -| Missing evidence | Show observed lower bound and telemetry/response coverage; missing usage unknown rather than non-use. | -| Gaming and limits | Invocation quotas and coercion; no target usage count, split server/client/self-report origins. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Self-report is not verified use; installation/loading is not use or benefit. Suppress small groups. | - - -### Change lead time - -Arithmetic mean commit-to-first-successful-production-deployment duration; also publish median/p90 and sample count. - -| Contract | Definition | -| --- | --- | -| Numerator | Sum of elapsed hours from committed_at to first successful production deployment for each service+commit | -| Denominator | Count of eligible service+commit pairs with valid mappings | -| Expression | N / D | -| Aggregation | mean | -| Unit | hours/change | -| Eligible population | Changes first successfully deployed to a service in reporting window; rollout retries collapse to same logical deployment. | -| Exclusions | Nonproduction, duplicates, no-op deployment with no new commits; missing mappings shown as coverage loss. | -| Events and source | change.committed; deployment.completed; VCS immutable commit timestamps and release manifest | -| Owner / cadence | Platform & IT / Weekly per service | -| Missing evidence | Missing commit mapping or negative/skewed duration excluded from numeric sample, reported explicitly; no zero substitution. | -| Gaming and limits | Timestamp rewriting and cherry-picking fast work; lock change mapping and show undeployed backlog age. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Not PR cycle time, effort saved, or a complete view of unfinished changes. p90 uses nearest-rank sorted sample. | - - -### Deployment frequency - -Distinct successful logical production deployments per complete service-day of observation. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct service_id+deployment_id successful production deployments in window | -| Denominator | Observed service-days in window | -| Expression | N / D | -| Aggregation | ratio | -| Unit | deployments/service-day | -| Eligible population | Declared production services and complete UTC observation days; report each service before pooled views. | -| Exclusions | Test/staging, failed attempts without production effect, duplicate callbacks and retries of the same rollout. | -| Events and source | deployment.completed; authoritative deployment service | -| Owner / cadence | Platform & IT / Weekly per service | -| Missing evidence | Missing feed periods not zero; report coverage and no unqualified overall rate when period incomplete. | -| Gaming and limits | Split deployments without value; pair with success, rework and failure measures. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. More deployments alone do not establish better performance; affected unsuccessful changes remain in change-fail population. | - - -### Change fail rate - -Production deployment effects requiring unplanned remediation, rollback or hotfix, not solely outage declarations. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct matured production deployment_ids linked to qualifying immediate intervention | -| Denominator | All distinct production deployment_ids that reached production in the matured deployment cohort | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Deployments with production effect during window and approved intervention follow-up horizon elapsed. | -| Exclusions | Failed preproduction attempts, duplicate callbacks and no-effect retries; failures after production effect stay included. | -| Events and source | deployment.completed; deployment.failed; incident.linked; release and incident records | -| Owner / cadence | Quality Engineering / Weekly; finalized after horizon | -| Missing evidence | Known failures / all eligible deployments is a lower bound where incident linkage incomplete; label provisional and report unknown links. | -| Gaming and limits | Avoid incident links or relabel intervention planned; reconcile rollbacks/hotfixes independently. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Define immediate intervention and follow-up horizon before pilot; no universal incident-to-AI attribution. | - - -### Failed deployment recovery time - -Mean elapsed time from production deployment failure onset to service restoration, with median/p90 and unresolved count/age. - -| Contract | Definition | -| --- | --- | -| Numerator | Sum of recovery_at minus failure_started_at in hours for recovered eligible deployment failures | -| Denominator | Count of recovered eligible deployment failures with valid timestamps | -| Expression | N / D | -| Aggregation | mean | -| Unit | hours/failure | -| Eligible population | Production deployment failures beginning in window; latest verified restoration if a recovery is rescinded. | -| Exclusions | Nondeployment incidents; invalid timestamp pairs reported missing, not treated as instantaneous recovery. | -| Events and source | deployment.failed; service.restored; incident service | -| Owner / cadence | Platform & IT / Weekly with unresolved failures | -| Missing evidence | Unresolved failures reported separately and excluded only from recovered-duration distribution; prevent declaring improvement without their visibility. | -| Gaming and limits | Premature recovery declarations or omit hard cases; require operational readback and carry unresolved ages. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Not all-incident MTTR; recovered-only sample has survivorship bias. p90 nearest rank. | - - -### Deployment rework rate - -Unplanned production deployment effects triggered by a production incident as a share of deployment effects. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct incident-linked deployment_ids classified unplanned remediation | -| Denominator | All distinct production deployment_ids with production effect in window | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Production deployment effects in window with immutable planned/unplanned classification and incident linkage. | -| Exclusions | Nonproduction, no-effect retries and duplicates; do not exclude unclassified changes silently. | -| Events and source | deployment.completed; incident.linked; release intent classification | -| Owner / cadence | Platform & IT / Weekly | -| Missing evidence | Unknown classification stays denominator; observed numerator is lower bound until reconciliation; show classification coverage. | -| Gaming and limits | Call hotfixes planned; freeze reason and independently reconcile incident/deployment links. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Distinct from failure rate: one failed deployment can lead to multiple separate rework deployments. | - - -### Changes with escaped quality issues - -Accepted changes with at least one confirmed post-acceptance defect during approved follow-up, split by severity. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct eligible change_ids with one or more linked confirmed defects in follow-up | -| Denominator | Accepted change_ids with completed predeclared follow-up horizon | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Accepted changes in acceptance window whose follow-up is complete by as_of. | -| Exclusions | Immature changes shown separately; duplicates/unconfirmed reports excluded but counted as pending triage. | -| Events and source | change.accepted; defect.confirmed; defect tracking and acceptance evidence | -| Owner / cadence | Quality Engineering / Monthly mature cohorts | -| Missing evidence | Unknown linkage is a coverage warning; no fabricated clean outcome. Show absolute defect and severe incident counts as well. | -| Gaming and limits | Shorten follow-up or downgrade severity; freeze horizon and rubric and independently sample defects. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Counts affected changes, not raw defects per change; no presumption defects are AI-caused. | - - -### Confirmed policy effects - -Eligible evaluated attempts with an actual confirmed prohibited effect; report attempts, blocked attempts and effects separately. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct attempt_ids with confirmed prohibited action effect | -| Denominator | All evaluated eligible attempt_ids including failed and abandoned runs | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Governed skill trials or supported workflow attempts, reported separately for eval and live populations. | -| Exclusions | Out-of-scope runs; detection gaps remain coverage issues, not exclusions from known initiated population. | -| Events and source | attempt.started; policy.violation; gate.blocked; security audit and incident triage | -| Owner / cadence | Architecture & Security / Immediate incident action; weekly aggregates | -| Missing evidence | Unknown outcomes remain visible; zero detected effects never proves safety. Missing audits make rate provisional. | -| Gaming and limits | Hide incidents or inflate harmless trials; absolute severity counts and fixed task mix accompany rate. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Blocked attempts are not actual effects; any critical effect blocks promotion regardless of aggregate rate. | - - -### Cost per accepted outcome - -Allocated total workflow cost including unsuccessful attempts, licenses, compute, tooling and human review/rework divided by accepted outcomes. - -| Contract | Definition | -| --- | --- | -| Numerator | Sum allocated costs in one approved reporting currency for all tasks/attempts in matured start cohort | -| Denominator | Distinct independently accepted tasks in same cohort | -| Expression | N / D | -| Aggregation | ratio | -| Unit | currency/accepted outcome | -| Eligible population | Matured eligible started-task cohort, with all attempt costs through as_of and allocated shared period costs. | -| Exclusions | Unrelated projects and documented taxes/transfers outside chosen accounting basis; never failed/abandoned attempts. | -| Events and source | cost.recorded; task.accepted; finance invoices, usage ledger, sampled effort with approved rates | -| Owner / cadence | Finance & Accounting Squad / Monthly with late invoice restatement | -| Missing evidence | Missing cost buckets produce partial estimate plus coverage, not a total. Mixed currency requires dated approved conversion; zero accepted outcomes = null with full cost shown. | -| Gaming and limits | Count tokens only or hide labor/failures; reconcile invoice and allocation totals without double counting. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Allocation and sampled labor carry uncertainty; no ROI without independently measured business benefit. | - - -### Verification confidence and cognitive load - -Three separate anonymous 1–5 item distributions: confidence in verifying output, manageable workload, usefulness. Primary item: verification confidence mean; report full distributions and response rate. - -| Contract | Definition | -| --- | --- | -| Numerator | Sum valid 1–5 verification-confidence responses, one per participant per survey wave | -| Denominator | Count valid responses to that item | -| Expression | N / D | -| Aggregation | mean | -| Unit | points (1–5) | -| Eligible population | Invited eligible cohort at fixed survey wave; voluntary responses. | -| Exclusions | Invalid/out-of-range answers and superseded duplicate submission; nonresponders stay in response-rate denominator. | -| Events and source | survey.response; anonymous survey with single-use token and aggregate invitation count | -| Owner / cadence | People & Administration Squad / After pilot workflow cycle; periodic follow-up | -| Missing evidence | Report valid response count / invited count and item nonresponse; no imputation. Suppress people cohorts below five. | -| Gaming and limits | Pressure to respond positively; no manager access to individual responses, avoid identifying free text. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Perception and selection bias; not measured time savings. No composite productivity index. | - - -### Skills within substantive review window - -Active approved skill revisions with an accountable owner and completed substantive review not past due. - -| Contract | Definition | -| --- | --- | -| Numerator | Distinct active approved skill revisions with current owner, review evidence and review_due_at >= as_of | -| Denominator | All active approved skill revisions in catalog snapshot | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | Approved revisions currently supported for use at as_of. | -| Exclusions | Experimental and retired revisions; deprecated/revoked in-use counts reported independently, not hidden. | -| Events and source | catalog.published; catalog.reviewed; catalog.deprecated; catalog.revoked; version inventory | -| Owner / cadence | Data & AI / Monthly and each release | -| Missing evidence | Missing owner/date/evidence outside numerator; show ownerless, deprecated-in-use and revoked-in-use absolute counts. | -| Gaming and limits | Rubber-stamp review dates; require linked regression/security/usefulness evidence. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. A current review is not proof of effectiveness; consumer inventory may miss shadow copies. | - - -### Workflow-specific business acceptance - -Independently accepted domain outcomes against a predeclared business rubric, separate from technical completion. - -| Contract | Definition | -| --- | --- | -| Numerator | Eligible completed-observation task_ids meeting the department-specific outcome rubric | -| Denominator | All eligible initiated workflow task_ids with matured business-outcome horizon | -| Expression | N / D * 100 | -| Aggregation | percent | -| Unit | % | -| Eligible population | One named departmental workflow and predeclared start cohort; never pool unlike rubrics. | -| Exclusions | Cancelled-before-start and unrelated workflows; unsuccessful or abandoned eligible work remains. | -| Events and source | task.started; business.accepted; departmental case/content/reconciliation/onboarding review record | -| Owner / cadence | Product Management / Each pilot cycle with owning department | -| Missing evidence | Unknown follow-up stays denominator with unresolved count; no claim actual business outcome changed without source evidence. | -| Gaming and limits | Weaken rubric or count drafts as customer benefit; lock rubric and independently audit acceptance. Descriptive signal, not causal AI productivity attribution. Source completeness, task mix and sampling limit inference; report uncertainty. Domain owner signs acceptance: Operations service accuracy; Marketing claims/rights; Finance reconciliation accuracy; HR policy correctness; Product usability/value. Not revenue uplift. | - - -## Run the synthetic measurement example - -**Executed synthetic example — not Eldorado results.** The bundled fixture produces 1 independently accepted task from 4 matured starts: 25% confirmed success. One unresolved task remains in that denominator; known-outcome coverage is 75%. Selection precision and recall are each 1 / 2 = 50%. Recorded synthetic compute cost is EUR 60 / 1 accepted outcome = EUR 60 per accepted outcome, a partial estimate: licenses, tooling, review and rework are missing. These deliberately small samples demonstrate arithmetic and missingness, not a baseline, target or ROI claim. - -From the private documentation checkout, run `node scripts/ai-kpi.mjs` and `node --test scripts/ai-kpi.test.mjs`. The fixture `scripts/fixtures/ai-kpi-synthetic.json` is labeled synthetic; every result carries the same label and calculation version. This prototype has no network collectors and does not enable employee telemetry. - -The executable covers validated success, selection precision/recall, change fail and rework rates, escaped quality, policy effects, cost and human experience. Other catalog metrics remain contracts rather than implemented collectors; output lists the omissions. Snapshot inputs are simplified, pre-reconciled records, not a production ingestion system. Conflicting duplicates reject the calculation rather than silently choosing a value. - -**Next:** [Use findings to maintain, quarantine or replace skills →](../skills-maintenance.mdx) - -
diff --git a/docs/strategy/roadmap.mdx b/docs/strategy/roadmap.mdx index 512b060..8079022 100644 --- a/docs/strategy/roadmap.mdx +++ b/docs/strategy/roadmap.mdx @@ -33,7 +33,7 @@ Repeat this loop within every phase—not as six global roadmap stages. Skills t **Understand the engineering path first:** codebase structure and existing principles → system-logic source → tests → builds and packaging → deployments. Inspect actual practices, tooling, controls and problems with the people doing the work, rather than assuming the documented process is the real one. -**Establish the comparison, then engineer skills.** Define metrics/KPIs and a baseline for comparable work before changing it. [Skills Measurement](./measurement.mdx) explains collection and calculation. Connect reviewed skills to the engineering path, with clear triggers, dependencies and acceptance checks. +**Establish the comparison, then engineer skills.** Define metrics/KPIs and a baseline for comparable work before changing it. [Skills Measurement](./measurement/index.mdx) explains collection and calculation. Connect reviewed skills to the engineering path, with clear triggers, dependencies and acceptance checks. **Use, collect, compare, repeat.** A human works with an AI Agent on real changes. Collect results, friction and feedback alongside quality, cost and verification effort. Compare with the baseline to understand the new current situation; revise the skills and working practices, then run the loop again. diff --git a/docs/strategy/skills-adoption.mdx b/docs/strategy/skills-adoption.mdx index 018fc07..036e2ba 100644 --- a/docs/strategy/skills-adoption.mdx +++ b/docs/strategy/skills-adoption.mdx @@ -36,6 +36,6 @@ No developer rankings, individual productivity scores, minimum invocation quotas -**Next:** [Measure accepted outcomes, not activity →](./measurement.mdx) +**Next:** [Measure accepted outcomes, not activity →](./measurement/index.mdx) diff --git a/docs/strategy/stage-01-skills-adoption.mdx b/docs/strategy/stage-01-skills-adoption.mdx index 6929a93..0a54ec5 100644 --- a/docs/strategy/stage-01-skills-adoption.mdx +++ b/docs/strategy/stage-01-skills-adoption.mdx @@ -28,7 +28,7 @@ Skills topology connects what skills exist, how they relate, when they are used - [Governance](./skills-governance.mdx) — define ownership, authority and approval of skills. - [Quality](./skills-quality.mdx) — qualify skills and verify their outcomes before wider use. - [Adoption](./skills-adoption.mdx) — coach bounded workflows with a supported client and a manual path. -- [Measurement](./measurement.mdx) — establish KPIs, collect evidence and compare outcomes with the baseline. +- [Measurement](./measurement/index.mdx) — establish KPIs, collect evidence and compare outcomes with the baseline. - [Maintenance](./skills-maintenance.mdx) — review feedback, control versions and recover from unsafe updates. Proven skills, measurements and working practices support [Stage - 02 - Autonomous Agent](./stage-02-autonomous-agent.mdx); adoption alone does not grant autonomous execution authority. diff --git a/docs/strategy/stage-02-autonomous-agent.mdx b/docs/strategy/stage-02-autonomous-agent.mdx index c7d4df8..1d999eb 100644 --- a/docs/strategy/stage-02-autonomous-agent.mdx +++ b/docs/strategy/stage-02-autonomous-agent.mdx @@ -13,7 +13,7 @@ This future stage of the [Roadmap](./roadmap.mdx) builds on [Stage - 01 - Skills ## Build on what has been proven -Carry forward proven skills, measurements and working practices from real human + AI Agent work. The skills foundation provides [governance](./skills-governance.mdx), [quality checks](./skills-quality.mdx), [adoption feedback](./skills-adoption.mdx), [measurement](./measurement.mdx) and [maintenance](./skills-maintenance.mdx); it is not permission to expand authority automatically. +Carry forward proven skills, measurements and working practices from real human + AI Agent work. The skills foundation provides [governance](./skills-governance.mdx), [quality checks](./skills-quality.mdx), [adoption feedback](./skills-adoption.mdx), [measurement](./measurement/index.mdx) and [maintenance](./skills-maintenance.mdx); it is not permission to expand authority automatically. ## Introduce authority deliberately diff --git a/docs/strategy/target-operating-model.mdx b/docs/strategy/target-operating-model.mdx index 7365911..97a7138 100644 --- a/docs/strategy/target-operating-model.mdx +++ b/docs/strategy/target-operating-model.mdx @@ -157,7 +157,7 @@ Success means accepted outcomes with maintained quality, controlled risk and und - [Skills governance](./skills-governance.mdx) — Contract, lifecycle, decision rights, adapters, locking, exceptions. - [Skills quality](./skills-quality.mdx) — Risk tiers, independent evaluation, repeated trials, adversarial conformance. - [Skills adoption](./skills-adoption.mdx) — Workflow-led coaching, client choice, feedback, manual path. -- [Measurement](./measurement.mdx) — Full KPI contracts, minimal events, illustrative calculations, privacy and limits. +- [Measurement](./measurement/index.mdx) — Full KPI contracts, minimal events, illustrative calculations, privacy and limits. - [Skills maintenance](./skills-maintenance.mdx) — Review, drift, deprecation, rollback and incident handling. diff --git a/package.json b/package.json index c9ad7cb..1e9691c 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "verify:build": "node scripts/verify.mjs --build", "diagrams:build": "node scripts/build-diagrams.mjs", "test": "node --test scripts/*.test.mjs", - "test:browser": "node scripts/browser/sidebar-coverage.cjs && node scripts/browser/skills-stories.cjs && node scripts/browser/strategy-navigation.cjs && node scripts/browser/strategy-story.cjs && node scripts/browser/inline-controls.cjs && node scripts/browser/sdlc-cycle.cjs && node scripts/browser/target-midpoints.cjs && node scripts/browser/skills-measurement.cjs && node scripts/browser/roadmap.cjs && node scripts/browser/roadmap-timeline.cjs", + "test:browser": "node scripts/browser/measurement-pages.cjs && node scripts/browser/sidebar-coverage.cjs && node scripts/browser/skills-stories.cjs && node scripts/browser/strategy-navigation.cjs && node scripts/browser/strategy-story.cjs && node scripts/browser/inline-controls.cjs && node scripts/browser/sdlc-cycle.cjs && node scripts/browser/target-midpoints.cjs && node scripts/browser/skills-measurement.cjs && node scripts/browser/roadmap.cjs && node scripts/browser/roadmap-timeline.cjs", "kpi:example": "node scripts/ai-kpi.mjs" }, "dependencies": { diff --git a/research/ai-adoption/measurement-migration.json b/research/ai-adoption/measurement-migration.json new file mode 100644 index 0000000..701f272 --- /dev/null +++ b/research/ai-adoption/measurement-migration.json @@ -0,0 +1,160 @@ +{ + "sourceCommit": "b9c2047c614dd4e825fa746815104d35d9500739", + "sourcePath": "docs/strategy/reference/measurement-contracts.mdx", + "metrics": [ + { + "oldAnchor": "validated-task-success", + "title": "Validated task success", + "docId": "measurement/validated-task-success", + "url": "/strategy/measurement/validated-task-success/", + "calculationId": "validated-task-success", + "status": "prototype", + "contractSha256": "925100f4c84f43e2cfdca4a89d6b4a984c808dae2d2ec5e57202318f70b9c544" + }, + { + "oldAnchor": "skill-selection-precision", + "title": "Skill selection precision", + "docId": "measurement/skill-selection-precision", + "url": "/strategy/measurement/skill-selection-precision/", + "calculationId": "selection-precision", + "status": "prototype", + "contractSha256": "189e93e4e5f52a7e70e2045c6b824a9da621382f39ba8ce9a59c0c04812f0ce3" + }, + { + "oldAnchor": "skill-selection-recall", + "title": "Skill selection recall", + "docId": "measurement/skill-selection-recall", + "url": "/strategy/measurement/skill-selection-recall/", + "calculationId": "selection-recall", + "status": "prototype", + "contractSha256": "847b5ba6094585eb88b8eb2b55b42e0f3c6822dcaa1629256ad0501a13eb7f18" + }, + { + "oldAnchor": "supported-bundle-coverage", + "title": "Supported bundle coverage", + "docId": "measurement/supported-bundle-coverage", + "url": "/strategy/measurement/supported-bundle-coverage/", + "calculationId": "supported-bundle-coverage", + "status": "unimplemented", + "contractSha256": "8b4dc2dbdfb6999fac283814f518645d0830d515125910d79dd7dc8763f0521c" + }, + { + "oldAnchor": "approved-workflow-adoption", + "title": "Approved-workflow adoption", + "docId": "measurement/approved-workflow-adoption", + "url": "/strategy/measurement/approved-workflow-adoption/", + "calculationId": "useful-adoption", + "status": "unimplemented", + "contractSha256": "355996d5adb603021ba22f9e1f7f08f2fbea017dac9ee0a4bd42cc00bf7adea3" + }, + { + "oldAnchor": "change-lead-time", + "title": "Change lead time", + "docId": "measurement/change-lead-time", + "url": "/strategy/measurement/change-lead-time/", + "calculationId": "change-lead-time", + "status": "unimplemented", + "contractSha256": "96fd0edd5eba139a7c5905b606bcc96e608c5294acd2ef0c962fdb7d339d8251" + }, + { + "oldAnchor": "deployment-frequency", + "title": "Deployment frequency", + "docId": "measurement/deployment-frequency", + "url": "/strategy/measurement/deployment-frequency/", + "calculationId": "deployment-frequency", + "status": "unimplemented", + "contractSha256": "5a34ba52e5c2768e29a1a7574274e07a65509f75063e89ccc7ce1b035aeff4c5" + }, + { + "oldAnchor": "change-fail-rate", + "title": "Change fail rate", + "docId": "measurement/change-fail-rate", + "url": "/strategy/measurement/change-fail-rate/", + "calculationId": "change-fail-rate", + "status": "prototype", + "contractSha256": "d2a4ecaddd468f42db5824f9bfab8a8c8fcdf9157b33b4e4f047269a83c8567b" + }, + { + "oldAnchor": "failed-deployment-recovery-time", + "title": "Failed deployment recovery time", + "docId": "measurement/failed-deployment-recovery-time", + "url": "/strategy/measurement/failed-deployment-recovery-time/", + "calculationId": "failed-deployment-recovery-time", + "status": "unimplemented", + "contractSha256": "489fd756883d4ccee1769fcfaa0b67725420463df2b1f6faf3e7faa7002f7a80" + }, + { + "oldAnchor": "deployment-rework-rate", + "title": "Deployment rework rate", + "docId": "measurement/deployment-rework-rate", + "url": "/strategy/measurement/deployment-rework-rate/", + "calculationId": "deployment-rework-rate", + "status": "prototype", + "contractSha256": "e8ade457136feac3ad39e9eba2a9d67142bc605727d9f4f038a7df6d5a0707fd" + }, + { + "oldAnchor": "changes-with-escaped-quality-issues", + "title": "Changes with escaped quality issues", + "docId": "measurement/changes-with-escaped-quality-issues", + "url": "/strategy/measurement/changes-with-escaped-quality-issues/", + "calculationId": "escaped-quality-rate", + "status": "prototype", + "contractSha256": "c436d504c8b3a9f655de18b2ca00fa8a0ab47ea1cf37637ff21561f81ecc77c9" + }, + { + "oldAnchor": "confirmed-policy-effects", + "title": "Confirmed policy effects", + "docId": "measurement/confirmed-policy-effects", + "url": "/strategy/measurement/confirmed-policy-effects/", + "calculationId": "policy-effects", + "status": "prototype", + "contractSha256": "b64c02f5516b08421fa8b816f0f490571afd05baf5ad6e9d92cb6a7b711ac82a" + }, + { + "oldAnchor": "cost-per-accepted-outcome", + "title": "Cost per accepted outcome", + "docId": "measurement/cost-per-accepted-outcome", + "url": "/strategy/measurement/cost-per-accepted-outcome/", + "calculationId": "cost-per-accepted-outcome", + "status": "prototype", + "contractSha256": "d8865ec31315d3c8398ec58cdb6665cb506a7d6309ebec3da0ca7dea84a5f406" + }, + { + "oldAnchor": "verification-confidence-and-cognitive-load", + "title": "Verification confidence and cognitive load", + "docId": "measurement/verification-confidence-and-cognitive-load", + "url": "/strategy/measurement/verification-confidence-and-cognitive-load/", + "calculationId": "human-experience", + "status": "prototype", + "contractSha256": "01e017471004c5f780fd0c2671a4d4449d392956cb4c5f0bd7791959d6a9af9a" + }, + { + "oldAnchor": "skills-within-substantive-review-window", + "title": "Skills within substantive review window", + "docId": "measurement/skills-within-substantive-review-window", + "url": "/strategy/measurement/skills-within-substantive-review-window/", + "calculationId": "maintenance-health", + "status": "unimplemented", + "contractSha256": "339fe0da6b382a34c52b5ec6005ce08a7293a2b07ccd88540a6df03a02d96ad4" + }, + { + "oldAnchor": "workflow-specific-business-acceptance", + "title": "Workflow-specific business acceptance", + "docId": "measurement/workflow-specific-business-acceptance", + "url": "/strategy/measurement/workflow-specific-business-acceptance/", + "calculationId": "department-outcome", + "status": "unimplemented", + "contractSha256": "55fce8843409a1bc1383fef3cee5b9a616b44fa8d45ba84c6e4ed490a25f2de4" + } + ], + "shared": [ + { + "slug": "collection-and-safeguards", + "sha256": "8610ed8df6a24cf138f9e6e2c09060db2aab4c0b1259ca0168e581e3fb964e52" + }, + { + "slug": "synthetic-example", + "sha256": "24af68ff4fd61159d1c0557ed0f8f8462470d56c6c83816eb93b47fb04149fde" + } + ] +} diff --git a/scripts/browser/measurement-pages.cjs b/scripts/browser/measurement-pages.cjs new file mode 100644 index 0000000..4c3c2ca --- /dev/null +++ b/scripts/browser/measurement-pages.cjs @@ -0,0 +1,79 @@ +const {chromium} = require('playwright'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +(async () => { + const base = process.argv[2] || 'http://127.0.0.1:4173'; + const prefix = '/eldorado-v1/documentation'; + const manifest = JSON.parse(fs.readFileSync('research/ai-adoption/measurement-migration.json','utf8')); + const output = `.cache/metric-pages-${process.argv[3] || 'local'}`; + fs.mkdirSync(output,{recursive:true}); + const browser = await chromium.launch({headless:true,executablePath:process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE,args:['--no-sandbox']}); + const results = []; + try { + for (const width of [1440,390]) { + const context = await browser.newContext({viewport:{width,height:900}}); + try { + const page = await context.newPage(); + const errors = []; + page.on('pageerror',e => errors.push(e.message)); + for (const m of manifest.metrics) { + const url = `${base}${prefix}${m.url}`; + assert.equal((await page.goto(url,{waitUntil:'networkidle'})).status(),200); + assert.equal(await page.locator('article h1').innerText(),m.title); + assert.equal(await page.locator('link[rel="canonical"]').getAttribute('href'),`https://pages.apps.lego-cloud.eu${prefix}${m.url}`); + const table = page.locator('article table'); + assert.equal(await table.count(),1); + assert.equal(await table.locator('tbody tr').count(),11); + const article = await page.locator('article').innerText(); + assert(article.includes(m.status === 'prototype' ? 'Prototype calculation' : 'Unimplemented metric')); + assert(article.includes('no live telemetry')); + assert.equal(await page.locator('.breadcrumbs a').filter({hasText:'Measurement'}).getAttribute('href'),`${prefix}/strategy/measurement/`); + assert(!await page.evaluate(() => document.documentElement.scrollWidth > innerWidth)); + await page.goto(`${base}${prefix}/strategy/reference/measurement-contracts/#${m.oldAnchor}`,{waitUntil:'networkidle'}); + await page.waitForURL(url); + await page.locator('article h1').waitFor(); + results.push({width,metric:m.oldAnchor,canonical:true,redirect:true,fields:11}); + } + for (const suffix of ['', '#unknown-anchor', '#__proto__']) { + await page.goto(`${base}${prefix}/strategy/reference/measurement-contracts/${suffix}`,{waitUntil:'networkidle'}); + await page.waitForURL(`${base}${prefix}/strategy/measurement/`); + } + for (const [anchor,target] of [ + ['collection-and-safeguards','collection-and-safeguards/'], + ['decide-what-counts-before-counting','collection-and-safeguards/#decide-what-counts-before-counting'], + ['baseline-and-interpretation','collection-and-safeguards/#baseline-and-interpretation'], + ['event-and-privacy-contract','collection-and-safeguards/#event-and-privacy-contract'], + ['run-the-synthetic-measurement-example','synthetic-example/'], + ['metric-contracts','#metric-definitions'], + ]) { + await page.goto(`${base}${prefix}/strategy/reference/measurement-contracts/#${anchor}`,{waitUntil:'networkidle'}); + await page.waitForURL(`${base}${prefix}/strategy/measurement/${target}`); + if (target.includes('#')) await page.locator(`#${target.split('#')[1]}`).waitFor(); + } + await page.goto(`${base}${prefix}/strategy/measurement/supported-bundle-coverage/`,{waitUntil:'networkidle'}); + await page.locator('.breadcrumbs a').filter({hasText:'Measurement'}).click(); + await page.waitForURL(`${base}${prefix}/strategy/measurement/`); + if (width < 997) await page.getByRole('button',{name:'Toggle navigation bar'}).click(); + const menu = page.locator('.theme-doc-sidebar-menu').filter({visible:true}); + const category = menu.locator('.theme-doc-sidebar-item-category').filter({has:page.getByRole('link',{name:'Measurement',exact:true})}).last(); + const caret = category.locator(':scope > .menu__list-item-collapsible > button.menu__caret'); + await caret.click(); + await category.getByRole('link',{name:'Validated task success',exact:true}).waitFor({state:'hidden'}); + await caret.focus(); await page.keyboard.press('Enter'); + await category.getByRole('link',{name:'Validated task success',exact:true}).waitFor({state:'visible'}); + await menu.getByRole('link',{name:'Measurement',exact:true}).click(); + await page.waitForURL(`${base}${prefix}/strategy/measurement/`); + await page.locator('article h1').waitFor(); + if (width < 997) await page.getByRole('button',{name:'Close navigation bar'}).waitFor({state:'hidden'}); + await page.locator('#metric-definitions').scrollIntoViewIfNeeded(); + await page.screenshot({path:`${output}/${width}-index.png`}); + await page.goto(`${base}${prefix}/strategy/measurement/supported-bundle-coverage/`,{waitUntil:'networkidle'}); + await page.screenshot({path:`${output}/${width}-metric.png`,fullPage:true}); + assert.deepEqual(errors,[]); + } finally {await context.close();} + } + assert.equal(results.length,32); + fs.writeFileSync(`${output}/results.json`,JSON.stringify(results,null,2)); + console.log('PASS: 16 metric pages × 2 widths; canonical metadata, contract fields, status, all legacy metric/shared/fallback redirects and linked Measurement keyboard navigation'); + } finally {await browser.close();} +})().catch(e => {console.error(e);process.exitCode=1;}); diff --git a/scripts/browser/sidebar-coverage.cjs b/scripts/browser/sidebar-coverage.cjs index 1a8825a..f118178 100644 --- a/scripts/browser/sidebar-coverage.cjs +++ b/scripts/browser/sidebar-coverage.cjs @@ -43,18 +43,19 @@ const fs = require('node:fs'); } const reference = '/eldorado-v1/documentation/strategy/reference/measurement-contracts/'; await page.goto(`${base}${reference}#supported-bundle-coverage`, {waitUntil:'networkidle'}); - await page.locator('#supported-bundle-coverage').waitFor(); + await page.waitForURL(`${base}/eldorado-v1/documentation/strategy/measurement/supported-bundle-coverage/`); + await page.getByRole('heading', {level:1, name:'Supported bundle coverage', exact:true}).waitFor(); assert((await page.locator('article').innerText()).includes('Supported bundle coverage')); assert.equal(await page.locator('.breadcrumbs a').filter({hasText:'Stage - 01 - Skills Adoption'}).getAttribute('href'), '/eldorado-v1/documentation/strategy/stage-01-skills-adoption/'); - assert((await page.locator('.breadcrumbs').innerText()).includes('Reference')); + assert((await page.locator('.breadcrumbs').innerText()).includes('Measurement')); await open(); - await menu.getByRole('link', {name:'Measurement Contracts', exact:true}).waitFor({state:'visible'}); + await menu.getByRole('link', {name:'Supported bundle coverage', exact:true}).waitFor({state:'visible'}); await page.screenshot({path:`${output}/${width}-reference.png`, animations:'disabled'}); assert.deepEqual(errors, []); } finally {await context.close();} } assert.equal(results.length, documents.length * 3); fs.writeFileSync(`${output}/results.json`, JSON.stringify(results,null,2)); - console.log(`PASS: all ${documents.length} docs have visible unique sidebar entries at 3 viewports; Contribution links, Reference breadcrumbs and reported anchor work`); + console.log(`PASS: all ${documents.length} docs have visible unique sidebar entries at 3 viewports; Contribution links, Measurement breadcrumbs and reported anchor redirect work`); } finally {await browser.close();} })().catch(e => {console.error(e); process.exitCode = 1;}); diff --git a/scripts/browser/skills-measurement.cjs b/scripts/browser/skills-measurement.cjs index 67ef003..70a6a73 100644 --- a/scripts/browser/skills-measurement.cjs +++ b/scripts/browser/skills-measurement.cjs @@ -18,11 +18,12 @@ const fs = require('node:fs'); assert.equal(await page.locator('h1').innerText(), 'Skills Measurement'); assert((await page.title()).startsWith('Skills Measurement |')); assert.equal(await page.locator('.strategy-content details').count(), 0); - assert.equal(await page.locator('.strategy-content h2').count(), 4); + assert.equal(await page.locator('.strategy-content h2').count(), 5); if (width < 700) await page.getByRole('button', {name:'Toggle navigation bar'}).click(); const menu = page.locator('.theme-doc-sidebar-menu .menu__link').filter({visible:true}); await menu.first().waitFor({state:'visible'}); - assert.deepEqual(await menu.allTextContents(), ['Target','Roadmap','Stage - 01 - Skills Adoption','Governance','Quality','Adoption','Measurement','Maintenance','Reference','Stage - 02 - Autonomous Agent']); + const metrics = JSON.parse(fs.readFileSync('research/ai-adoption/measurement-migration.json','utf8')).metrics; + assert.deepEqual(await menu.allTextContents(), ['Target','Roadmap','Stage - 01 - Skills Adoption','Governance','Quality','Adoption','Measurement',...metrics.map(m => m.title),'Collection and safeguards','Run the synthetic measurement example','Maintenance','Stage - 02 - Autonomous Agent']); if (width < 700) await page.getByRole('button', {name:'Close navigation bar'}).click(); const panel = page.locator('.archify-panel--story'); const iframe = panel.locator(width > 700 ? '.story-desktop' : '.story-mobile'); @@ -65,15 +66,14 @@ const fs = require('node:fs'); await page.locator('.strategy-content h2').nth(1).scrollIntoViewIfNeeded(); await page.screenshot({path:`${output}/${width}-pilot.png`}); await page.screenshot({path:`${output}/${width}-full.png`, fullPage:true}); - const links = await page.locator('.strategy-content a[href*="measurement-contracts"]').evaluateAll(nodes => nodes.map(n => n.href)); + const links = await page.locator('.strategy-content a[href*="/measurement/"]').evaluateAll(nodes => nodes.map(n => n.href)); for (const href of [...new Set(links)]) { const navigation = await page.goto(href, {waitUntil:'networkidle'}); if (navigation) assert.equal(navigation.status(), 200); assert.equal(page.url(), href); if (new URL(href).hash) await page.locator(new URL(href).hash).waitFor({state:'visible'}); } - assert.equal(await page.locator('.strategy-content h3').count(), 19); - assert((await page.locator('.strategy-content').innerText()).includes('9 prototype calculations and 7 unimplemented metrics')); + assert.equal(new Set(links.map(h => new URL(h).pathname)).size, 18); assert(!await page.evaluate(() => document.documentElement.scrollWidth > innerWidth)); assert.deepEqual(errors, []); results.push({width, geometry, links:links.length, title:true, sidebar:true, controls:true, export:true, reference:true}); diff --git a/scripts/browser/strategy-navigation.cjs b/scripts/browser/strategy-navigation.cjs index 91cd980..7426048 100644 --- a/scripts/browser/strategy-navigation.cjs +++ b/scripts/browser/strategy-navigation.cjs @@ -8,9 +8,12 @@ const fs = require('node:fs'); const prefix = '/eldorado-v1/documentation/strategy/'; const first = 'Stage - 01 - Skills Adoption'; const second = 'Stage - 02 - Autonomous Agent'; - const topics = ['Governance','Quality','Adoption','Measurement','Maintenance','Reference']; - const ids = ['target-operating-model','roadmap','stage-01-skills-adoption','skills-governance','skills-quality','skills-adoption','measurement','skills-maintenance','reference/measurement-contracts','stage-02-autonomous-agent']; - const titles = ['Target','Roadmap',first,'Skills governance','Skills quality','Skills adoption','Skills Measurement','Skills maintenance','Skills KPI reference',second]; + const topics = ['Governance','Quality','Adoption','Measurement','Maintenance']; + const metrics = JSON.parse(fs.readFileSync('research/ai-adoption/measurement-migration.json','utf8')).metrics; + const childIds = [...metrics.map(m => m.docId),'measurement/collection-and-safeguards','measurement/synthetic-example']; + const childTitles = [...metrics.map(m => m.title),'Collection and safeguards','Run the synthetic measurement example']; + const ids = ['target-operating-model','roadmap','stage-01-skills-adoption','skills-governance','skills-quality','skills-adoption','measurement',...childIds,'skills-maintenance','stage-02-autonomous-agent']; + const titles = ['Target','Roadmap',first,'Skills governance','Skills quality','Skills adoption','Skills Measurement',...childTitles,'Skills maintenance',second]; const browser = await chromium.launch({headless:true, executablePath:process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE, args:['--no-sandbox']}); const results = []; try { @@ -39,11 +42,12 @@ const fs = require('node:fs'); await category.getByRole('link', {name:'Governance', exact:true}).waitFor({state:'visible'}); assert.deepEqual(await visibleLabels(), ['Target','Roadmap',first,...topics,second]); assert.deepEqual(await category.locator('ul .menu__link').filter({visible:true}).allTextContents(), topics); - const reference = category.getByRole('button', {name:'Reference', exact:true}); + const measurement = category.locator('.theme-doc-sidebar-item-category').filter({has:page.getByRole('link', {name:'Measurement', exact:true})}); + const reference = measurement.locator(':scope > .menu__list-item-collapsible > button.menu__caret'); await reference.focus(); await page.keyboard.press('Enter'); - await category.getByRole('link', {name:'Measurement Contracts', exact:true}).waitFor({state:'visible'}); - assert.deepEqual(await visibleLabels(), ['Target','Roadmap',first,...topics,'Measurement Contracts',second]); + await category.getByRole('link', {name:'Validated task success', exact:true}).waitFor({state:'visible'}); + assert.deepEqual(await visibleLabels(), ['Target','Roadmap',first,...topics.slice(0,4),...childTitles,'Maintenance',second]); await reference.click(); await page.screenshot({path:`${output}/${width}-expanded.png`, animations:'disabled'}); await caret.click(); @@ -68,10 +72,12 @@ const fs = require('node:fs'); else assert.equal(await prev.count(), 0); if (index < ids.length-1) assert.equal(await next.getAttribute('href'), `${prefix}${ids[index+1]}/`); else assert.equal(await next.count(), 0); - if (index >= 3 && index <= 7) { + if (index >= 3 && index < ids.length - 1) { assert.equal(await page.locator('.breadcrumbs a').filter({hasText:first}).getAttribute('href'), `${prefix}stage-01-skills-adoption/`); menu = await openMenu(); - assert.deepEqual(await visibleLabels(), ['Target','Roadmap',first,...topics,second]); + const expectedTopics = id === 'measurement' || id.startsWith('measurement/') ? [...topics.slice(0,4),...childTitles,'Maintenance'] : topics; + assert.deepEqual(await visibleLabels(), ['Target','Roadmap',first,...expectedTopics,second]); + if (id.startsWith('measurement/')) assert.equal(await page.locator('.breadcrumbs a').filter({hasText:'Measurement'}).getAttribute('href'), `${prefix}measurement/`); await closeMenu(); } assert(!await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), `${width}: overflow on ${id}`); @@ -84,6 +90,6 @@ const fs = require('node:fs'); } finally {await context.close();} } assert.equal(results.length, 4); - console.log('PASS: 4 desktop/mobile viewports; all 10 routes; linked category, keyboard expansion, nested Reference, breadcrumbs, next/previous and no overflow/errors'); + console.log(`PASS: 4 desktop/mobile viewports; all ${ids.length} routes; linked Measurement category, keyboard expansion, breadcrumbs, next/previous and no overflow/errors`); } finally {await browser.close();} })().catch(error => {console.error(error); process.exitCode = 1;}); diff --git a/scripts/browser/strategy-story.cjs b/scripts/browser/strategy-story.cjs index 57547da..25db9f0 100644 --- a/scripts/browser/strategy-story.cjs +++ b/scripts/browser/strategy-story.cjs @@ -24,7 +24,7 @@ const fs=require('node:fs'); assert.equal(geometry.nodes.length,(source.components||source.nodes).length,id); for(const n of geometry.nodes) assert(n.x>=0 && n.y>=0 && n.right<=geometry.width+1 && n.bottom<=geometry.height+1,`${id} ${width} clipped ${JSON.stringify(n)} / ${geometry.height}`); const data=await page.locator('.strategy-content').evaluate(el=>({words:el.innerText.trim().split(/\s+/).length,overflow:document.documentElement.scrollWidth>innerWidth,openDetails:el.querySelectorAll('details[open]').length})); - assert(data.words<=(id==='measurement'?1100:id==='roadmap'?600:200),id);assert(!data.overflow,id);assert.equal(data.openDetails,0,id); + assert(data.words<=(id==='measurement'?1500:id==='roadmap'?600:200),id);assert(!data.overflow,id);assert.equal(data.openDetails,0,id); await page.evaluate(()=>scrollTo({top:0,behavior:'instant'}));await page.screenshot({path:`.cache/story-${label}/${width}-${id}.png`,fullPage:true}); const opener=panel.getByRole('button',{name:/^Expand /});await opener.click(); const close=panel.getByRole('button',{name:'Close full-screen diagram',exact:true});await close.waitFor({state:'visible'}); @@ -44,7 +44,8 @@ const fs=require('node:fs'); results.push({id,width,...data,geometry,modalClick:true,modalEscape:true,focusRestored:true,details:id!=='measurement'});fs.writeFileSync(`.cache/story-${label}/results.json`,JSON.stringify(results,null,2)); } await page.goto(`${base}/eldorado-v1/documentation/strategy/reference/measurement-contracts/#validated-task-success`,{waitUntil:'networkidle'}); - const heading=page.locator('#validated-task-success');await heading.waitFor({state:'visible'}); + await page.waitForURL(`${base}/eldorado-v1/documentation/strategy/measurement/validated-task-success/`); + await page.getByRole('heading',{level:1,name:'Validated task success',exact:true}).waitFor(); assert.equal(await page.locator('.strategy-content details').count(),0,'reference is directly readable'); await context.close(); } diff --git a/scripts/measurement-pages.mjs b/scripts/measurement-pages.mjs new file mode 100644 index 0000000..885405f --- /dev/null +++ b/scripts/measurement-pages.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import {readFileSync, readdirSync} from 'node:fs'; +import {runInNewContext} from 'node:vm'; +const read = p => readFileSync(p, 'utf8'); + +export function verifyMeasurementPages() { + const manifest = JSON.parse(read('research/ai-adoption/measurement-migration.json')); + const {documents, exclusions} = JSON.parse(read('.cache/sidebar-coverage.json')); + const prefix = '/eldorado-v1/documentation'; + const legacy = 'strategy/reference/measurement-contracts/index.html'; + assert.equal(exclusions.filter(e => e.file === legacy).length, 1); + assert(!documents.some(d => d.id === 'reference/measurement-contracts')); + const search = readdirSync('build').filter(f => /^search-index.*\.json$/.test(f)).map(f => read(`build/${f}`)).join('\n'); + assert(!search.includes('/strategy/reference/measurement-contracts/')); + const children = [...manifest.metrics.map(m => ({...m,slug:m.oldAnchor})), + {slug:'collection-and-safeguards',title:'Collection and safeguards'}, + {slug:'synthetic-example',title:'Run the synthetic measurement example'}, + ]; + for (const child of children) { + const route = `${prefix}/strategy/measurement/${child.slug}/`; + const doc = documents.filter(d => `${d.route}/` === route); + assert.equal(doc.length,1); + assert.equal(doc[0].id, `measurement/${child.slug}`); + const page = read(`build/strategy/measurement/${child.slug}/index.html`); + const canonical = [...page.matchAll(/]*rel="canonical"[^>]*>/g)]; + assert.equal(canonical.length,1); + assert(canonical[0][0].includes(`href="https://pages.apps.lego-cloud.eu${route}"`)); + assert(search.includes(route), `Missing search route: ${route}`); + assert(!page.includes(`${prefix}/strategy/reference/measurement-contracts/`)); + } + const script = read(`build/${legacy}`).match(/Continue to Skills Measurement