565 lines
40 KiB
JSON
565 lines
40 KiB
JSON
[
|
||
{
|
||
"id": 1,
|
||
"title": "Agent Skills specification",
|
||
"url": "https://agentskills.io/specification",
|
||
"publisher": "Agent Skills",
|
||
"retrieved_at": "2026-10-04T10:27:53.189577+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "The specification defines a skill directory with SKILL.md, not a universal deployment or enforcement service.",
|
||
"evidence": "A skill is a directory containing, at minimum, a SKILL.md file:",
|
||
"limitations": "Format compatibility does not certify discovery, precedence, tool permissions, safety, or instruction compliance."
|
||
},
|
||
{
|
||
"claim": "allowed-tools is experimental in the common specification.",
|
||
"evidence": "allowed-toolsNoSpace-separated string of pre-approved tools the skill may use. (Experimental)",
|
||
"limitations": "Format compatibility does not certify discovery, precedence, tool permissions, safety, or instruction compliance."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/01.txt",
|
||
"body_sha256": "0913e84055bc7eb05649aa49123a783dc1a413e4c31d0861f9ac367849c7b00e",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 2,
|
||
"title": "Copilot Agent Skills",
|
||
"url": "https://docs.github.com/en/copilot/concepts/agents/about-agent-skills",
|
||
"publisher": "GitHub",
|
||
"retrieved_at": "2026-10-04T10:27:53.333904+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "GitHub documents skill support across specified Copilot surfaces, not all modes indiscriminately.",
|
||
"evidence": "Agent skills work with Copilot cloud agent, Copilot code review, the GitHub Copilot CLI, the GitHub Copilot app, and agent mode in Visual Studio Code and JetBrains IDEs.",
|
||
"limitations": "Vendor documentation, not an Eldorado installation test. Separate adding-skills page omits JetBrains in its overview; validate exact host/version."
|
||
},
|
||
{
|
||
"claim": "Copilot documents multiple project skill paths.",
|
||
"evidence": "Project skills, stored in your repository (.github/skills, .claude/skills, or .agents/skills)",
|
||
"limitations": "Vendor documentation, not an Eldorado installation test. Separate adding-skills page omits JetBrains in its overview; validate exact host/version."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/02.txt",
|
||
"body_sha256": "26c86bc2afa4813c46361429f015f687164a69f39030fa05dbc1f513a04c0ca3",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 3,
|
||
"title": "Copilot custom instructions",
|
||
"url": "https://docs.github.com/en/copilot/concepts/prompting/response-customization",
|
||
"publisher": "GitHub",
|
||
"retrieved_at": "2026-10-04T10:27:53.960265+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "GitHub explicitly warns that custom-instruction following is nondeterministic.",
|
||
"evidence": "Due to the non-deterministic nature of AI, Copilot may not always follow your custom instructions in exactly the same way every time they are used.",
|
||
"limitations": "Instruction precedence is surface-specific and is not enforceable authorization. Head-branch instructions create a review trust risk."
|
||
},
|
||
{
|
||
"claim": "The GitHub.com instruction hierarchy prioritizes personal over repository over organization instructions.",
|
||
"evidence": "Multiple types of custom instructions can apply to a request sent to Copilot. Personal instructions take the highest priority. Repository instructions come next, and then organization instructions are prioritized last. However, all sets of relevant instructions are provided to Copilot.",
|
||
"limitations": "Instruction precedence is surface-specific and is not enforceable authorization. Head-branch instructions create a review trust risk."
|
||
},
|
||
{
|
||
"claim": "Copilot review reads instructions and skills from the PR head branch.",
|
||
"evidence": "When reviewing a pull request, Copilot reads repository custom instructions, agent instructions, and agent skills from the head branch (the branch with your changes), not the base branch. For example, when merging my-feature-branch into main, Copilot uses the instructions and skills in my-feature-branch, so you can test changes to them in the same pull request without merging them first.",
|
||
"limitations": "Instruction precedence is surface-specific and is not enforceable authorization. Head-branch instructions create a review trust risk."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/03.txt",
|
||
"body_sha256": "1c2c190052653b8baad1478dc53b7a0657b741be0dbb0ef202b3f503c6b3379c",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 4,
|
||
"title": "Claude Code skills",
|
||
"url": "https://code.claude.com/docs/en/skills",
|
||
"publisher": "Anthropic",
|
||
"retrieved_at": "2026-10-04T10:27:53.088978+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Claude skill name collisions prioritize enterprise over personal over project.",
|
||
"evidence": "Two of enterprise, personal, and projectEnterprise over personal, and personal over project. With deploy in both ~/.claude/skills/ and the project’s .claude/skills/, /deploy runs the personal one",
|
||
"limitations": "Vendor extensions and changing versions prevent assuming cross-client parity. No local client exercised."
|
||
},
|
||
{
|
||
"claim": "Claude extends the common skill format with client-specific functionality.",
|
||
"evidence": "Claude Code skills follow the Agent Skills open standard, which works across multiple AI tools. Claude Code extends the standard with additional features like invocation control, subagent execution, and dynamic context injection. See Using skill frontmatter outside Claude Code for which frontmatter fields are part of the standard and which are Claude Code extensions.",
|
||
"limitations": "Vendor extensions and changing versions prevent assuming cross-client parity. No local client exercised."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/04.txt",
|
||
"body_sha256": "4ac3de187f2ba050da149e1bea3364a959366be03963e7a76dc9f9041bdb4e1f",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 5,
|
||
"title": "Claude Code memory",
|
||
"url": "https://code.claude.com/docs/en/memory",
|
||
"publisher": "Anthropic",
|
||
"retrieved_at": "2026-10-04T10:27:52.994293+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Claude memory instructions are context, not enforced configuration.",
|
||
"evidence": "Claude Code has two complementary memory systems. Both are loaded at the start of every conversation. Claude treats them as context, not enforced configuration. To block an action regardless of what Claude decides, use a PreToolUse hook instead. The more specific and concise your instructions, the more consistently Claude follows them.",
|
||
"limitations": "Context order does not establish deterministic behavior; conflicting prose may still produce arbitrary choices."
|
||
},
|
||
{
|
||
"claim": "Claude concatenates discovered memory files rather than treating them as override-only configuration.",
|
||
"evidence": "All discovered files are concatenated into context rather than overriding each other. Across the directory tree, content is ordered from the filesystem root down to your working directory. For the foo/bar/ example, foo/CLAUDE.md appears in context before foo/bar/CLAUDE.md, so instructions closer to where you launched Claude are read last. Within each directory, CLAUDE.local.md is appended after CLAUDE.md, so your personal notes are the last thing Claude reads at that level.",
|
||
"limitations": "Context order does not establish deterministic behavior; conflicting prose may still produce arbitrary choices."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/05.txt",
|
||
"body_sha256": "b19869f25886f95f8d93cec941482cbf9472b507ab8d643b446b1450b915e0df",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 6,
|
||
"title": "Claude Code hooks",
|
||
"url": "https://code.claude.com/docs/en/hooks",
|
||
"publisher": "Anthropic",
|
||
"retrieved_at": "2026-10-04T10:27:53.059984+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Claude PreToolUse command/http/MCP hook timeout is not fail-closed.",
|
||
"evidence": "A timed-out command, http, or mcp_tool hook doesn’t block the tool call. The call continues through the normal permission flow, so don’t count on a stalled hook to act as a gate.",
|
||
"limitations": "Event and hook-family distinctions matter; no universal fail-closed promise. No hook was installed or tested here."
|
||
},
|
||
{
|
||
"claim": "A conventional exit code 1 alone does not reliably block most Claude hook events.",
|
||
"evidence": "For most hook events, exit code 2 is the only exit code that blocks through the code alone. Without valid JSON on stdout, Claude Code treats exit code 1 as a non-blocking error and proceeds with the action, even though 1 is the conventional Unix failure code. If your hook is meant to enforce a policy, use exit 2. The worktree events differ: any non-zero exit code from WorktreeCreate aborts worktree creation, and any non-zero exit code from WorktreeRemove makes worktree removal fail if the directory still exists afterward.",
|
||
"limitations": "Event and hook-family distinctions matter; no universal fail-closed promise. No hook was installed or tested here."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/06.txt",
|
||
"body_sha256": "ae1919a5b06aa6d4a213bfafcaab1d2473f0b89d53b9fcc35358a19ab9ca3e5d",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 7,
|
||
"title": "Claude Code security",
|
||
"url": "https://code.claude.com/docs/en/security",
|
||
"publisher": "Anthropic",
|
||
"retrieved_at": "2026-10-04T10:27:52.953904+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Claude working-directory approval is not OS-level containment.",
|
||
"evidence": "Working directory boundary: In Manual mode, Claude Code asks you before its file tools read or write outside the folder where it was started and its subfolders. The boundary is a permission prompt, so a Bash command you approve can still write anywhere your user account can",
|
||
"limitations": "Vendor capability is not evidence the sandbox is enabled; approved shell commands inherit actual OS authority."
|
||
},
|
||
{
|
||
"claim": "OS-level Bash restrictions require sandboxing.",
|
||
"evidence": "To restrict Bash commands at the operating system level, turn on sandboxing",
|
||
"limitations": "Vendor capability is not evidence the sandbox is enabled; approved shell commands inherit actual OS authority."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/07.txt",
|
||
"body_sha256": "a8a2ad7d077347f1d3ea43549b0be1070e03c7737647f73ef40035d5dafb5f40",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 8,
|
||
"title": "Cline skills",
|
||
"url": "https://docs.cline.bot/customization/skills",
|
||
"publisher": "Cline",
|
||
"retrieved_at": "2026-10-04T10:27:53.250798+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Cline global skill overrides same-name project skill.",
|
||
"evidence": "When a global skill and project skill have the same name, the global skill takes precedence. This lets you keep general-purpose skills globally while using project-specific skills in .cline/skills/ so the whole team can use them.",
|
||
"limitations": "Current docs disagree with Agent Skills guide generalization. Exact installed Cline version remains unknown."
|
||
},
|
||
{
|
||
"claim": "Discovered Cline skills have user-controlled enable/disable toggles.",
|
||
"evidence": "Every skill has a toggle to enable or disable it. This lets you control which skills are active without deleting the skill directory. Skills are enabled by default when discovered.",
|
||
"limitations": "Current docs disagree with Agent Skills guide generalization. Exact installed Cline version remains unknown."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/08.txt",
|
||
"body_sha256": "38e6af22c7e9f78e90e899a7bd285e6556bfd64197fbc3f7cf279cc9ff29d416",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 9,
|
||
"title": "Cline rules",
|
||
"url": "https://docs.cline.bot/customization/cline-rules",
|
||
"publisher": "Cline",
|
||
"retrieved_at": "2026-10-04T10:27:53.308915+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Cline workspace rules override conflicting global rules, unlike skill name precedence.",
|
||
"evidence": "When both workspace and global rules exist, Cline combines them. Workspace rules take precedence when they conflict with global rules. See Storage Locations for more guidance.",
|
||
"limitations": "Rule precedence is not skill precedence; both are guidance and can be toggled."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/09.txt",
|
||
"body_sha256": "c1a2353c024491a42961e5ea171c341c66bbed2647464d200bf90fdbaa8a48e3",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 10,
|
||
"title": "Cline hooks",
|
||
"url": "https://docs.cline.bot/customization/hooks",
|
||
"publisher": "Cline",
|
||
"retrieved_at": "2026-10-04T10:27:53.370475+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Cline hooks landing page delegates details to SDK plugins.",
|
||
"evidence": "See details under SDK Plugins page.",
|
||
"limitations": "Landing page alone proves no event schema or IDE compatibility. SDK page was followed separately."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/10.txt",
|
||
"body_sha256": "2ebd8cd1c0b11fec466a3b577971382d2c4ec1f9911679b10100029e9b3be095",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 11,
|
||
"title": "Demystifying evals for AI agents",
|
||
"url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents",
|
||
"publisher": "Anthropic",
|
||
"retrieved_at": "2026-10-04T10:27:53.451618+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Repeated trials are needed for variable agent outputs.",
|
||
"evidence": "Each attempt at a task is a trial. Because model outputs vary between runs, we run multiple trials to produce more consistent results.",
|
||
"limitations": "Vendor-authored methodology and examples, not independent proof of any skill gain or target threshold."
|
||
},
|
||
{
|
||
"claim": "Agent outcome evaluation must inspect environment state rather than trust claimed completion.",
|
||
"evidence": "The outcome is the final state in the environment at the end of the trial. A flight-booking agent might say “Your flight has been booked” at the end of the transcript, but the outcome is whether a reservation exists in the environment’s SQL database.",
|
||
"limitations": "Vendor-authored methodology and examples, not independent proof of any skill gain or target threshold."
|
||
},
|
||
{
|
||
"claim": "Agent evals combine code, model and human graders.",
|
||
"evidence": "Agent evaluations typically combine three types of graders: code-based, model-based, and human. Each grader evaluates some portion of either the transcript or the outcome. An essential component of effective evaluation design is to choose the right graders for the job.",
|
||
"limitations": "Vendor-authored methodology and examples, not independent proof of any skill gain or target threshold."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/11.txt",
|
||
"body_sha256": "7940a33d8e19ff68aac5e3c095e0fa771565a4b2416e8a781e9ed02d55668a12",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 12,
|
||
"title": "DORA software delivery metrics",
|
||
"url": "https://dora.dev/guides/dora-metrics/",
|
||
"publisher": "DORA",
|
||
"retrieved_at": "2026-10-04T10:27:53.335018+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "DORA now documents five software-delivery performance metrics.",
|
||
"evidence": "These metrics have evolved alongside the technology landscape: shifting from the original four keys to the current five-metric model. To understand the research behind changes like the move from MTTR to Failed Deployment Recovery Time, read A history of DORA’s software delivery metrics.",
|
||
"limitations": "Service/team outcome measures, not causal AI-effect estimates or individual productivity scores."
|
||
},
|
||
{
|
||
"claim": "DORA change lead time runs from commit to production deployment.",
|
||
"evidence": "Change lead time: The amount of time it takes for a change to go from committed to version control to deployed in production.",
|
||
"limitations": "Service/team outcome measures, not causal AI-effect estimates or individual productivity scores."
|
||
},
|
||
{
|
||
"claim": "DORA deployment rework measures unplanned incident-driven deployments.",
|
||
"evidence": "Deployment rework rate: The ratio of deployments that are unplanned but happen as a result of an incident in production.",
|
||
"limitations": "Service/team outcome measures, not causal AI-effect estimates or individual productivity scores."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/12.txt",
|
||
"body_sha256": "0535efe22d238f38d1de5ec655efeb44a313e59f8a71085e902bde5c7a693cb9",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 13,
|
||
"title": "NIST AI RMF",
|
||
"url": "https://www.nist.gov/itl/ai-risk-management-framework",
|
||
"publisher": "NIST",
|
||
"retrieved_at": "2026-10-04T10:27:53.731461+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "NIST AI RMF is voluntary guidance for trustworthiness across the AI lifecycle.",
|
||
"evidence": "Led by the Information Technology Laboratory (ITL) AI Program, and in collaboration with the private and public sectors, NIST has developed a framework to better manage risks to individuals, organizations, and society associated with artificial intelligence (AI). The NIST AI Risk Management Framework (AI RMF) is intended for voluntary use and to improve the ability to incorporate trustworthiness considerations into the design, development, use, and evaluation of AI products, services, and systems.",
|
||
"limitations": "Voluntary framework, not certification, legal advice, or evidence of Eldorado compliance."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/14.txt",
|
||
"body_sha256": "2fed4e9811f05579384a23b9c3b81600ddf0f011fc4cfe638352a817f91ed603",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 14,
|
||
"title": "NIST Generative AI Profile",
|
||
"url": "https://www.nist.gov/publications/artificial-intelligence-risk-management-framework-generative-artificial-intelligence",
|
||
"publisher": "NIST",
|
||
"retrieved_at": "2026-10-04T10:27:53.743664+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "NIST GenAI profile is a cross-sectoral companion to AI RMF.",
|
||
"evidence": "This document is a cross-sectoral profile of and companion resource for the AI Risk Management Framework (AI RMF 1.0) for Generative AI, pursuant to President Biden's Executive Order (EO) 14110 on Safe, Secure, and Trustworthy Artificial Intelligence. The AI RMF was released in January 2023, and is intended for voluntary use and to improve the ability of organizations to incorporate trustworthiness considerations into the design, development, use, and evaluation of AI products, services, and systems.",
|
||
"limitations": "Publication landing-page abstract inspected; full PDF not read. Do not attribute detailed risk taxonomy to this excerpt."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/15.txt",
|
||
"body_sha256": "7998580f4dc83770568dc361ecae1d46e1d652a2df12a278263b88de6cbfb624",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 15,
|
||
"title": "OWASP prompt injection",
|
||
"url": "https://genai.owasp.org/llmrisk/llm01-prompt-injection/",
|
||
"publisher": "OWASP",
|
||
"retrieved_at": "2026-10-04T10:27:53.529810+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "External sources can carry prompt injections.",
|
||
"evidence": "Indirect prompt injections occur when an LLM accepts input from external sources, such as websites or files. The content may have in the external content data that when interpreted by the model, alters the behavior of the model in unintended or unexpected ways. Like direct injections, indirect injections can be either intentional or unintentional.",
|
||
"limitations": "Mitigations reduce exposure; no guarantee that prompt injection is eliminated."
|
||
},
|
||
{
|
||
"claim": "OWASP recommends least privilege.",
|
||
"evidence": "Provide the application with its own API tokens for extensible functionality, and handle these functions in code rather than providing them to the model. Restrict the model’s access privileges to the minimum necessary for its intended operations.",
|
||
"limitations": "Mitigations reduce exposure; no guarantee that prompt injection is eliminated."
|
||
},
|
||
{
|
||
"claim": "OWASP recommends human approval for high-risk operations.",
|
||
"evidence": "Implement human-in-the-loop controls for privileged operations to prevent unauthorized actions.",
|
||
"limitations": "Mitigations reduce exposure; no guarantee that prompt injection is eliminated."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/16.txt",
|
||
"body_sha256": "5bfcc42a422165d0a6b25f28fbff144fc0dec5289f1a672a40203e1d64200c07",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 16,
|
||
"title": "Experienced developer RCT",
|
||
"url": "https://metr.org/blog/2025-07-10-early-2025-ai-experienced-os-dev-study/",
|
||
"publisher": "METR",
|
||
"retrieved_at": "2026-10-04T10:27:54.289507+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "METR early-2025 RCT found 19% longer task time in its setting.",
|
||
"evidence": "We conduct a randomized controlled trial (RCT) to understand how early-2025 AI tools affect the productivity of experienced open-source developers working on their own repositories. Surprisingly, we find that when developers use AI tools, they take 19% longer than without—AI makes them slower. We view this result as a snapshot of early-2025 AI capabilities in one relevant setting; as these systems continue to rapidly evolve, we plan on continuing to use this methodology to help estimate AI acceleration from AI R&D automation.",
|
||
"limitations": "Historical narrow setting, tools from early 2025; superseded as a current estimate by authors. No Eldorado forecast."
|
||
},
|
||
{
|
||
"claim": "Study involved 16 experienced developers and 246 issues on their repositories.",
|
||
"evidence": "To directly measure the real-world impact of AI tools on software development, we recruited 16 experienced developers from large open-source repositories (averaging 22k+ stars and 1M+ lines of code) that they’ve contributed to for multiple years. Developers provide lists of real issues (246 total) that would be valuable to the repository—bug fixes, features, and refactors that would normally be part of their regular work. Then, we randomly assign each issue to either allow or disallow use of AI while working on the issue. When AI is allowed, developers can use any tools they choose (primarily Cursor Pro with Claude 3.5/3.7 Sonnet—frontier models at the time of the study); when disallowed, they work without generative AI assistance. Developers complete these tasks (which average two hours each) while recording their screens, then self-report the total implementation time they needed. We pay developers $150/hr as compensation for their participation in the study.",
|
||
"limitations": "Historical narrow setting, tools from early 2025; superseded as a current estimate by authors. No Eldorado forecast."
|
||
},
|
||
{
|
||
"claim": "METR now explicitly marks those historical results as not current.",
|
||
"evidence": "We have released results that are current as of early 2026, in a continuation of this study. We believe these historical results no longer reflect the current impact of AI models on open-source developer productivity.",
|
||
"limitations": "Historical narrow setting, tools from early 2025; superseded as a current estimate by authors. No Eldorado forecast."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/17.txt",
|
||
"body_sha256": "f9abab8a3ad312279800d8327122ebae70c13804d9f91fda1788db1a466a178b",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 17,
|
||
"title": "AI developer productivity experiment",
|
||
"url": "https://arxiv.org/abs/2302.06590",
|
||
"publisher": "Peng et al.",
|
||
"retrieved_at": "2026-10-04T10:27:53.477297+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Peng et al. controlled HTTP-server task study reports 55.8% faster completion with Copilot.",
|
||
"evidence": "Abstract:Generative AI tools hold promise to increase human productivity. This paper presents results from a controlled experiment with GitHub Copilot, an AI pair programmer. Recruited software developers were asked to implement an HTTP server in JavaScript as quickly as possible. The treatment group, with access to the AI pair programmer, completed the task 55.8% faster than the control group. Observed heterogenous effects show promise for AI pair programmers to help people transition into software development careers.",
|
||
"limitations": "Primary paper abstract inspected, not full methods. Bounded JavaScript task; not whole-SDLC productivity."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/18.txt",
|
||
"body_sha256": "76c71ee9844c5066d184ec5006945c48959dfcd75854842463cd756f348a3140",
|
||
"evidence_type": "primary research abstract"
|
||
},
|
||
{
|
||
"id": 18,
|
||
"title": "How much does AI impact development speed? An enterprise-based randomized controlled trial",
|
||
"url": "https://arxiv.org/abs/2410.12944",
|
||
"publisher": "Paradis et al. / arXiv",
|
||
"retrieved_at": "2026-10-04T10:27:53.495104+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Paradis et al. Google RCT abstract reports about 21% shorter task time with a large confidence interval, among 96 engineers.",
|
||
"evidence": "Abstract:How much does AI assistance impact developer productivity? To date, the software engineering literature has provided a range of answers, targeting a diversity of outcomes: from perceived productivity to speed on task and developer throughput. Our randomized controlled trial with 96 full-time Google software engineers contributes to this literature by sharing an estimate of the impact of three AI features on the time developers spent on a complex, enterprise-grade task. We found that AI significantly shortened the time developers spent on task. Our best estimate of the size of this effect, controlling for factors known to influence developer time on task, stands at about 21\\%, although our confidence interval is large. We also found an interesting effect whereby developers who spend more hours on code-related activities per day were faster with AI. Product and future research considerations are discussed. In particular, we invite further research that explores the impact of AI at the ecosystem level and across multiple suites of AI-enhanced tools, since we cannot assume that the effect size obtained in our lab study will necessarily apply more broadly, or that the effect of AI found using internal Google tooling in the summer of 2024 will translate across tools and over time.",
|
||
"limitations": "Primary paper abstract inspected, not full methods. Google-specific task/tool setting; no universal gain claim. Retrieval seed title was wrong and corrected from body."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/19.txt",
|
||
"body_sha256": "e3ac83e7a8c9a613ed67b179321599538d4132e0caf858c4d335452c924c6839",
|
||
"evidence_type": "primary research abstract"
|
||
},
|
||
{
|
||
"id": 19,
|
||
"title": "Copilot metrics concepts",
|
||
"url": "https://docs.github.com/en/copilot/concepts/billing-and-usage/copilot-usage-metrics/copilot-metrics",
|
||
"publisher": "GitHub",
|
||
"retrieved_at": "2026-10-04T10:27:54.384631+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Copilot richest metrics depend on enabled client telemetry.",
|
||
"evidence": "Copilot usage metrics are derived from telemetry across multiple Copilot surfaces, including IDE, Copilot CLI, and agent apps activity. Most metrics come from client-side IDE telemetry, and end users need telemetry enabled in their IDE for the richest data in these metrics.",
|
||
"limitations": "No access to Eldorado subscription, settings, enabled exporters, reports or coverage. Adoption is not impact."
|
||
},
|
||
{
|
||
"claim": "Server-side active-user coverage does not ensure complete feature/LOC breakdowns.",
|
||
"evidence": "Users surfaced through server-side telemetry are fully counted toward your active user totals (such as daily active users, daily_active_users ). When available, they may also appear in totals_by_ide (including the most recently detected IDE and Copilot extension versions in per-user reports). However, other dimensional breakdowns—such as totals_by_feature and lines-of-code metrics—remain empty until richer telemetry is available for them. Top-level totals and breakdowns for users already captured by client telemetry are unchanged.",
|
||
"limitations": "No access to Eldorado subscription, settings, enabled exporters, reports or coverage. Adoption is not impact."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/20.txt",
|
||
"body_sha256": "556fe5be28952b3b7d980b86b8a1a15713f0cff8359a7a0e02e09af7d6916ace",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 20,
|
||
"title": "Claude Code monitoring",
|
||
"url": "https://code.claude.com/docs/en/monitoring-usage",
|
||
"publisher": "Anthropic",
|
||
"retrieved_at": "2026-10-04T10:27:53.636289+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Claude supports configured OTel usage/cost/tool telemetry.",
|
||
"evidence": "Track Claude Code usage, costs, and tool activity across your organization by exporting telemetry data through OpenTelemetry (OTel). Claude Code exports metrics as time series data via the standard metrics protocol, events via the logs/events protocol, and optionally distributed traces via the traces protocol.",
|
||
"limitations": "Capability depends on version/configuration. Tool/skill-name export may expose confidential identifiers; no observed Eldorado telemetry."
|
||
},
|
||
{
|
||
"claim": "Skill-name/tool detail collection has explicit opt-in and privacy implications.",
|
||
"evidence": "OTEL_LOG_TOOL_DETAILSEnable logging of tool parameters and input arguments in tool events and trace span attributes: Bash commands, MCP server and tool names, skill names, user-authored workflow names, and tool input. Also enables custom, plugin, and MCP command names on user_prompt events, and real agent, skill, plugin, and MCP server and tool names on the cost and token counters (default: disabled). For Claude Desktop’s built-in servers, in sessions Claude Desktop owns, mcp_server_name/mcp_tool_name emit on tool_decision/tool_result even with the flag off. The exception requires Claude Code v2.1.214 or later1 to enable",
|
||
"limitations": "Capability depends on version/configuration. Tool/skill-name export may expose confidential identifiers; no observed Eldorado telemetry."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/21.txt",
|
||
"body_sha256": "197b3be58cae62470e82ac24074ca81024b030701ab3e2729116067dac798b48",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 21,
|
||
"title": "Agent Skills security",
|
||
"url": "https://agentskills.io/client-implementation/adding-skills-support",
|
||
"publisher": "Agent Skills",
|
||
"retrieved_at": "2026-10-04T10:27:53.937245+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "The implementation guide claims project-over-user skill precedence as universal.",
|
||
"evidence": "The universal convention across existing implementations: project-level skills override user-level skills.",
|
||
"limitations": "Implementation guidance rather than normative format; universal precedence claim contradicted by current Claude and Cline docs."
|
||
},
|
||
{
|
||
"claim": "Agent Skills format does not mandate discovery paths.",
|
||
"evidence": "The .agents/skills/ paths have emerged as a widely-adopted convention for cross-client skill sharing. While the Agent Skills specification does not mandate where skill directories live (it only defines what goes inside them), scanning .agents/skills/ means skills installed by other compliant clients are automatically visible to yours, and vice versa.",
|
||
"limitations": "Implementation guidance rather than normative format; universal precedence claim contradicted by current Claude and Cline docs."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/22.txt",
|
||
"body_sha256": "fb2909a93d445a4b18a7cab89c22a7e92502e99d2caa586cf785e8aa8ae7a77b",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 22,
|
||
"title": "METR productivity update 2026",
|
||
"url": "https://metr.org/blog/2026-02-24-uplift-update/",
|
||
"publisher": "METR",
|
||
"retrieved_at": "2026-10-04T10:29:00.438454+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "METR 2026 follow-up calls its productivity signal unreliable because of selection and time-measurement issues.",
|
||
"evidence": "Unfortunately, given participant feedback and surveys, we believe that the data from our new experiment gives us an unreliable signal of the current productivity effect of AI tools. The primary reason is that we have observed a significant increase in developers choosing not to participate in the study because they do not wish to work without AI, which likely biases downwards our estimate of AI-assisted speedup. We additionally believe there have been selection effects due to a lower pay rate (we reduced the pay from $150/hr to $50/hr), and that our measurements of time-spent on each task are unreliable for the fraction of developers who use multiple AI agents concurrently.",
|
||
"limitations": "Authors explicitly warn selection and parallel-agent timing compromise causal magnitude. Neither raw estimate is an Eldorado baseline."
|
||
},
|
||
{
|
||
"claim": "2026 raw estimates suggest speedup, but intervals include no effect and selection limits generalization.",
|
||
"evidence": "Our raw results show some evidence for speedup. Our early 2025 study found the use of AI causes tasks to take 19% longer, with a confidence interval between +2% and +39%. For the subset of the original developers who participated in the later study, we now estimate a speedup of -18% with a confidence interval between -38% and +9%. Among newly-recruited developers the estimated speedup is -4%, with a confidence interval between -15% and +9%.",
|
||
"limitations": "Authors explicitly warn selection and parallel-agent timing compromise causal magnitude. Neither raw estimate is an Eldorado baseline."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/23.txt",
|
||
"body_sha256": "7a4ce4ac6ba1a7a2410e17982493ab0d905d5d6418f28dda7460ca2aa4068306",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 23,
|
||
"title": "Cline SDK plugins",
|
||
"url": "https://docs.cline.bot/sdk/plugins",
|
||
"publisher": "Cline",
|
||
"retrieved_at": "2026-10-04T10:28:59.766735+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "Cline SDK plugins document fail_closed hook policy.",
|
||
"evidence": "Use fail_closed for policy-enforcement hooks where bypassing the hook is unsafe.",
|
||
"limitations": "SDK capability cannot be assumed identical in every Cline IDE/CLI build. Fail-closed behavior needs adapter conformance tests."
|
||
},
|
||
{
|
||
"claim": "Cline plugin installation accepts npm/git/file sources.",
|
||
"evidence": "Plugins can also be installed from file URLs, npm, git, or local paths using cline plugin install. See Plugins for install commands, the manifest format, and directory layout.",
|
||
"limitations": "SDK capability cannot be assumed identical in every Cline IDE/CLI build. Fail-closed behavior needs adapter conformance tests."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/24.txt",
|
||
"body_sha256": "445ba1089e4157e9eac938d1b7f1adf482769f3ae6d495d4bef5d666614fbd6a",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 24,
|
||
"title": "SPACE research publication",
|
||
"url": "https://www.microsoft.com/en-us/research/publication/the-space-of-developer-productivity-theres-more-to-it-than-you-think/",
|
||
"publisher": "Microsoft Research",
|
||
"retrieved_at": "2026-10-04T10:29:02.189612+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "SPACE rejects productivity measurement with a single activity metric.",
|
||
"evidence": "Developer productivity is about more than an individual’s activity levels or the efficiency of the engineering systems relied on to ship software, and it cannot be measured by a single metric or dimension. The SPACE framework captures different dimensions of productivity, and here we demonstrate how this framework can be used to understand productivity in practice and why using it will help teams better understand developer productivity and create better measures to inform their work and teams.",
|
||
"limitations": "Author-hosted publication summary retrieved after ACM 403; supports multidimensional principle, not detailed instrument validity."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/25.txt",
|
||
"body_sha256": "1f8176ad5b14a5f4053162f670b95bdef81e191104f682c506b11099525b734e",
|
||
"evidence_type": "primary research abstract"
|
||
},
|
||
{
|
||
"id": 25,
|
||
"title": "NIST AI RMF Core",
|
||
"url": "https://airc.nist.gov/airmf-resources/airmf/5-sec-core/",
|
||
"publisher": "NIST",
|
||
"retrieved_at": "2026-10-04T10:29:00.075579+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "NIST RMF organizes risk activity into govern, map, measure, manage.",
|
||
"evidence": "highest level to govern, map, measure, and manage AI risks. Governance is designed to be a cross-cutting function",
|
||
"limitations": "Risk-framework recommendations are not proof controls are deployed or staffing exists."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/26.txt",
|
||
"body_sha256": "c80bf3d3d59511eb3932cfd285893aa2a329ca9d997939c7c8c7718d704f857b",
|
||
"evidence_type": "primary source body"
|
||
},
|
||
{
|
||
"id": 26,
|
||
"title": "Copilot adding skills",
|
||
"url": "https://docs.github.com/en/copilot/how-tos/copilot-on-github/customize-copilot/customize-cloud-agent/add-skills",
|
||
"publisher": "GitHub",
|
||
"retrieved_at": "2026-10-04T10:29:00.472587+00:00",
|
||
"claims": [
|
||
{
|
||
"claim": "gh skill --pin skips pinned skills during updates.",
|
||
"evidence": "To lock a skill to a specific version (or commit SHA) so it is skipped during updates, use --pin:",
|
||
"limitations": "gh skill is public preview, requires documented minimum CLI version; Gitea interoperability not verified. Pinning is not artifact safety."
|
||
},
|
||
{
|
||
"claim": "GitHub warns shared skills are unverified and may be malicious.",
|
||
"evidence": "Skills are not verified by GitHub and may contain prompt injections, hidden instructions, or malicious scripts. Always inspect the content of a skill before installation using gh skill preview.",
|
||
"limitations": "gh skill is public preview, requires documented minimum CLI version; Gitea interoperability not verified. Pinning is not artifact safety."
|
||
},
|
||
{
|
||
"claim": "gh skill records source ref and tree SHA in frontmatter.",
|
||
"evidence": "When you install a skill with gh skill, provenance metadata is written into the skill's SKILL.md frontmatter, including the source repository, ref, and tree SHA. The gh skill update command uses this metadata to check for upstream changes.",
|
||
"limitations": "gh skill is public preview, requires documented minimum CLI version; Gitea interoperability not verified. Pinning is not artifact safety."
|
||
}
|
||
],
|
||
"evidence_file": "/opt/data/channels/eldorado-storm-v1/research/ai-adoption/bodies/27.txt",
|
||
"body_sha256": "7a5d8597f2ed72edb4610124e83f2abc282311739eabb561b3997ab6cbcbc46b",
|
||
"evidence_type": "primary source body"
|
||
}
|
||
]
|