[verified] feat: add safe source importers
This commit is contained in:
@@ -0,0 +1,745 @@
|
||||
import { constants } from "node:fs";
|
||||
import { open, readdir, realpath } from "node:fs/promises";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { isAbsolute, relative, resolve } from "node:path";
|
||||
|
||||
import { JSON_SCHEMA, loadAll } from "js-yaml";
|
||||
|
||||
import type { DiagramEdge, DiagramIRV2, DiagramNode } from "../../model/diagram-ir.js";
|
||||
import { validateDiagramIR } from "../../model/diagram-ir.js";
|
||||
|
||||
export type SourceKind = "python" | "javascript" | "typescript" | "go" | "rust" | "terraform" | "kubernetes" | "docker-compose" | "sql" | "openapi" | "ci";
|
||||
export type SourceInput = { type: "text"; text: string } | { type: "object"; value: unknown } | { type: "file"; root: string };
|
||||
export interface SourceImportRequest { sourceKind: SourceKind; path: string; input: SourceInput; maxBytes?: number }
|
||||
export interface ImportDiagnostic { code: string; severity: "warning" | "error"; message: string; path: string; line?: number }
|
||||
export interface SourceImportResult { diagram: DiagramIRV2; diagnostics: ImportDiagnostic[]; provenance: { sourceKind: SourceKind; path: string } }
|
||||
|
||||
function id(prefix: string, ...values: string[]): string {
|
||||
const encoded = values.map((value) => Buffer.from(value, "utf8").toString("hex") || "0");
|
||||
return `${prefix}-${encoded.join(".")}`;
|
||||
}
|
||||
|
||||
function codePointCompare(left: string, right: string): number {
|
||||
const leftPoints = [...left];
|
||||
const rightPoints = [...right];
|
||||
const length = Math.min(leftPoints.length, rightPoints.length);
|
||||
for (let index = 0; index < length; index += 1) {
|
||||
const difference = leftPoints[index].codePointAt(0)! - rightPoints[index].codePointAt(0)!;
|
||||
if (difference !== 0) return difference;
|
||||
}
|
||||
return leftPoints.length - rightPoints.length;
|
||||
}
|
||||
|
||||
function diagramFor(request: SourceImportRequest, nodes: DiagramNode[], edges: DiagramEdge[]): DiagramIRV2 {
|
||||
return validateDiagramIR({ version: 2, title: request.path, pages: [{ id: "source-import", title: request.path, nodes, edges, layout: { type: "layered", direction: "horizontal" } }], provenance: { sourceKind: request.sourceKind, path: request.path } }) as DiagramIRV2;
|
||||
}
|
||||
|
||||
function codeImports(kind: "python" | "go", text: string): Array<{ name: string; line: number }> {
|
||||
const imports: Array<{ name: string; line: number }> = [];
|
||||
let goBlock = false;
|
||||
text.split(/\r?\n/).forEach((line, index) => {
|
||||
let name: string | undefined;
|
||||
if (kind === "python") {
|
||||
const match = line.match(/^\s*(?:import\s+([A-Za-z_][\w.]*)|from\s+([A-Za-z_][\w.]*)\s+import\s+)/);
|
||||
name = match?.[1] ?? match?.[2];
|
||||
} else if (kind === "go") {
|
||||
if (/^\s*import\s*\(\s*$/.test(line)) { goBlock = true; return; }
|
||||
if (goBlock && /^\s*\)/.test(line)) { goBlock = false; return; }
|
||||
const match = goBlock ? line.match(/^\s*(?:[._A-Za-z]\w*\s+)?"([^"]+)"/) : line.match(/^\s*import\s+(?:[._A-Za-z]\w*\s+)?"([^"]+)"/);
|
||||
name = match?.[1];
|
||||
}
|
||||
if (name) imports.push({ name, line: index + 1 });
|
||||
});
|
||||
return imports;
|
||||
}
|
||||
|
||||
interface MaskedSource { text: string; unknownLines: number[] }
|
||||
|
||||
function maskRange(characters: string[], start: number, end: number): void {
|
||||
for (let index = start; index < end; index += 1) if (characters[index] !== "\n" && characters[index] !== "\r") characters[index] = " ";
|
||||
}
|
||||
|
||||
function maskPythonTripleStrings(text: string): MaskedSource {
|
||||
const characters = text.split("");
|
||||
const unknownLines: number[] = [];
|
||||
let offset = 0; let line = 1;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
||||
const quote = text.startsWith("'''", offset) ? "'''" : text.startsWith('"""', offset) ? '"""' : undefined;
|
||||
if (!quote) { offset += 1; continue; }
|
||||
const start = offset; const startLine = line;
|
||||
offset += 3;
|
||||
let closed = false;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
||||
if (text[offset] === "\n") line += 1;
|
||||
if (text.startsWith(quote, offset)) { offset += 3; closed = true; break; }
|
||||
offset += 1;
|
||||
}
|
||||
maskRange(characters, start, offset);
|
||||
if (!closed) unknownLines.push(startLine);
|
||||
}
|
||||
return { text: characters.join(""), unknownLines };
|
||||
}
|
||||
|
||||
function maskComments(text: string, lineMarkers: string[], nestedBlock = false, maskedQuotes = "", rustRawStrings = false): MaskedSource {
|
||||
const characters = text.split("");
|
||||
const unknownLines: number[] = [];
|
||||
let offset = 0; let line = 1;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
||||
const marker = lineMarkers.find((candidate) => text.startsWith(candidate, offset));
|
||||
if (marker) {
|
||||
const start = offset;
|
||||
while (offset < text.length && text[offset] !== "\n") offset += 1;
|
||||
maskRange(characters, start, offset);
|
||||
continue;
|
||||
}
|
||||
if (text.startsWith("/*", offset)) {
|
||||
const start = offset; const startLine = line;
|
||||
offset += 2;
|
||||
let depth = 1;
|
||||
while (offset < text.length && depth > 0) {
|
||||
if (text[offset] === "\n") line += 1;
|
||||
if (nestedBlock && text.startsWith("/*", offset)) { depth += 1; offset += 2; continue; }
|
||||
if (text.startsWith("*/", offset)) { depth -= 1; offset += 2; continue; }
|
||||
offset += 1;
|
||||
}
|
||||
maskRange(characters, start, offset);
|
||||
if (depth > 0) unknownLines.push(startLine);
|
||||
continue;
|
||||
}
|
||||
if (rustRawStrings && text[offset] === "r") {
|
||||
let delimiterEnd = offset + 1;
|
||||
while (text[delimiterEnd] === "#") delimiterEnd += 1;
|
||||
if (text[delimiterEnd] === '"') {
|
||||
const start = offset; const startLine = line;
|
||||
const terminator = `"${"#".repeat(delimiterEnd - offset - 1)}`;
|
||||
offset = delimiterEnd + 1;
|
||||
const rawEnd = text.indexOf(terminator, offset);
|
||||
const closed = rawEnd >= 0;
|
||||
const next = closed ? rawEnd + terminator.length : text.length;
|
||||
line += (text.slice(offset, next).match(/\n/g) ?? []).length;
|
||||
offset = next;
|
||||
maskRange(characters, start, offset);
|
||||
if (!closed) unknownLines.push(startLine);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (text[offset] === "'" || text[offset] === '"' || text[offset] === "`") {
|
||||
const quote = text[offset]; const start = offset; const startLine = line;
|
||||
offset += 1;
|
||||
let closed = false;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\\" && quote !== "`" && offset + 1 < text.length) { offset += 2; continue; }
|
||||
if (quote === "'" && text[offset] === "'" && text[offset + 1] === "'") { offset += 2; continue; }
|
||||
if (text[offset] === "\n") line += 1;
|
||||
if (text[offset] === quote) { offset += 1; closed = true; break; }
|
||||
offset += 1;
|
||||
}
|
||||
if (maskedQuotes.includes(quote)) maskRange(characters, start, offset);
|
||||
if (!closed) unknownLines.push(startLine);
|
||||
continue;
|
||||
}
|
||||
offset += 1;
|
||||
}
|
||||
return { text: characters.join(""), unknownLines };
|
||||
}
|
||||
|
||||
function maskTerraformHeredocs(text: string): MaskedSource {
|
||||
const characters = text.split("");
|
||||
const unknownLines: number[] = [];
|
||||
const heredocOpening = /<<(-?)([A-Za-z_]\w*)[ \t]*(?:\r?\n)/y;
|
||||
let offset = 0; let line = 1;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
||||
if (text[offset] === "#" || text.startsWith("//", offset)) {
|
||||
while (offset < text.length && text[offset] !== "\n") offset += 1;
|
||||
continue;
|
||||
}
|
||||
if (text.startsWith("/*", offset)) {
|
||||
const end = text.indexOf("*/", offset + 2);
|
||||
const next = end < 0 ? text.length : end + 2;
|
||||
line += (text.slice(offset, next).match(/\n/g) ?? []).length;
|
||||
offset = next;
|
||||
continue;
|
||||
}
|
||||
if (text[offset] === '"') {
|
||||
offset += 1;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
||||
if (text[offset] === "\n") line += 1;
|
||||
if (text[offset] === '"') { offset += 1; break; }
|
||||
offset += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
heredocOpening.lastIndex = offset;
|
||||
const opening = heredocOpening.exec(text);
|
||||
if (!opening) { offset += 1; continue; }
|
||||
const start = offset; const startLine = line;
|
||||
const indented = opening[1] === "-";
|
||||
const delimiter = opening[2];
|
||||
offset += opening[0].length;
|
||||
line += 1;
|
||||
let closed = false;
|
||||
while (offset < text.length) {
|
||||
const newline = text.indexOf("\n", offset);
|
||||
const end = newline < 0 ? text.length : newline;
|
||||
const candidate = text.slice(offset, end).replace(/\r$/, "");
|
||||
const expected = indented ? candidate.trim() : candidate.trimEnd();
|
||||
if (expected === delimiter && (indented || candidate === expected)) {
|
||||
offset = end;
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
offset = newline < 0 ? text.length : newline + 1;
|
||||
if (newline >= 0) line += 1;
|
||||
}
|
||||
maskRange(characters, start, offset);
|
||||
if (!closed) unknownLines.push(startLine);
|
||||
}
|
||||
return { text: characters.join(""), unknownLines };
|
||||
}
|
||||
|
||||
function maskTerraformStrings(text: string): MaskedSource {
|
||||
const characters = text.split("");
|
||||
const unknownLines: number[] = [];
|
||||
let offset = 0; let line = 1;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
||||
if (text[offset] !== '"') { offset += 1; continue; }
|
||||
const start = offset; const startLine = line;
|
||||
offset += 1;
|
||||
let closed = false;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
||||
if (text[offset] === "\n") line += 1;
|
||||
if (text[offset] === '"') { offset += 1; closed = true; break; }
|
||||
offset += 1;
|
||||
}
|
||||
maskRange(characters, start, offset);
|
||||
if (!closed) unknownLines.push(startLine);
|
||||
}
|
||||
return { text: characters.join(""), unknownLines };
|
||||
}
|
||||
|
||||
interface JavaScriptToken { kind: "identifier" | "string" | "punctuation"; value: string; line: number }
|
||||
|
||||
function scanJavaScriptImports(text: string): { imports: Array<{ name: string; line: number }>; unknownLines: number[] } {
|
||||
const tokens: JavaScriptToken[] = [];
|
||||
const unknownLines = new Set<number>();
|
||||
let offset = 0;
|
||||
let line = 1;
|
||||
while (offset < text.length) {
|
||||
const character = text[offset];
|
||||
if (character === "\n") { line += 1; offset += 1; continue; }
|
||||
if (/\s/.test(character)) { offset += 1; continue; }
|
||||
if (character === "/" && text[offset + 1] === "/") {
|
||||
offset += 2;
|
||||
while (offset < text.length && text[offset] !== "\n") offset += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "/" && text[offset + 1] === "*") {
|
||||
const startLine = line;
|
||||
offset += 2;
|
||||
let closed = false;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\n") line += 1;
|
||||
if (text[offset] === "*" && text[offset + 1] === "/") { offset += 2; closed = true; break; }
|
||||
offset += 1;
|
||||
}
|
||||
if (!closed) unknownLines.add(startLine);
|
||||
continue;
|
||||
}
|
||||
if (character === "/") {
|
||||
const previous = tokens[tokens.length - 1];
|
||||
const followsArrow = previous?.value === ">" && tokens[tokens.length - 2]?.value === "=";
|
||||
const regexPrefix = !previous || followsArrow || (previous.kind === "punctuation" && "=([{,:;!&|?".includes(previous.value)) || (previous.kind === "identifier" && new Set(["return", "case", "throw", "typeof", "instanceof", "in", "of", "yield", "await"]).has(previous.value));
|
||||
if (regexPrefix) {
|
||||
const startLine = line;
|
||||
let inCharacterClass = false;
|
||||
let closed = false;
|
||||
offset += 1;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
||||
if (text[offset] === "\n") break;
|
||||
if (text[offset] === "[") inCharacterClass = true;
|
||||
else if (text[offset] === "]") inCharacterClass = false;
|
||||
else if (text[offset] === "/" && !inCharacterClass) { offset += 1; closed = true; break; }
|
||||
offset += 1;
|
||||
}
|
||||
while (closed && offset < text.length && /[A-Za-z]/.test(text[offset])) offset += 1;
|
||||
if (!closed) unknownLines.add(startLine);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (character === "'" || character === '"') {
|
||||
const quote = character;
|
||||
const startLine = line;
|
||||
let value = "";
|
||||
offset += 1;
|
||||
let closed = false;
|
||||
while (offset < text.length) {
|
||||
const child = text[offset];
|
||||
if (child === "\\" && offset + 1 < text.length) { value += child + text[offset + 1]; offset += 2; continue; }
|
||||
if (child === quote) { offset += 1; closed = true; break; }
|
||||
if (child === "\n") line += 1;
|
||||
value += child;
|
||||
offset += 1;
|
||||
}
|
||||
if (closed) tokens.push({ kind: "string", value, line: startLine });
|
||||
else unknownLines.add(startLine);
|
||||
continue;
|
||||
}
|
||||
if (character === "`") {
|
||||
const startLine = line;
|
||||
let closed = false;
|
||||
let hasExpression = false;
|
||||
offset += 1;
|
||||
while (offset < text.length) {
|
||||
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
||||
if (text[offset] === "$" && text[offset + 1] === "{") hasExpression = true;
|
||||
if (text[offset] === "\n") line += 1;
|
||||
if (text[offset] === "`") { offset += 1; closed = true; break; }
|
||||
offset += 1;
|
||||
}
|
||||
if (hasExpression || !closed) unknownLines.add(startLine);
|
||||
continue;
|
||||
}
|
||||
if (/[A-Za-z_$]/.test(character)) {
|
||||
const start = offset;
|
||||
offset += 1;
|
||||
while (offset < text.length && /[A-Za-z0-9_$]/.test(text[offset])) offset += 1;
|
||||
tokens.push({ kind: "identifier", value: text.slice(start, offset), line });
|
||||
continue;
|
||||
}
|
||||
tokens.push({ kind: "punctuation", value: character, line });
|
||||
offset += 1;
|
||||
}
|
||||
|
||||
const imports: Array<{ name: string; line: number }> = [];
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
const token = tokens[index];
|
||||
if (token.kind !== "identifier") continue;
|
||||
if (token.value === "require" && tokens[index - 1]?.value !== "." && tokens[index + 1]?.value === "(") {
|
||||
const argument = tokens[index + 2];
|
||||
if (argument?.kind === "string" && tokens[index + 3]?.value === ")") imports.push({ name: argument.value, line: token.line });
|
||||
else unknownLines.add(token.line);
|
||||
continue;
|
||||
}
|
||||
if (token.value !== "import" && token.value !== "export") continue;
|
||||
if (tokens[index - 1]?.value === ".") continue;
|
||||
const next = tokens[index + 1];
|
||||
if (token.value === "import" && next?.value === "(") { unknownLines.add(token.line); continue; }
|
||||
const previous = tokens[index - 1];
|
||||
const statementPosition = !previous || previous.value === ";" || previous.value === "}" || previous.line < token.line;
|
||||
if (!statementPosition) continue;
|
||||
if (token.value === "import" && next?.kind === "string") { imports.push({ name: next.value, line: token.line }); continue; }
|
||||
let found = false;
|
||||
for (let cursor = index + 1; cursor < tokens.length && cursor <= index + 256; cursor += 1) {
|
||||
if (tokens[cursor].value === ";") break;
|
||||
if (tokens[cursor].value === "from" && tokens[cursor + 1]?.kind === "string") {
|
||||
imports.push({ name: tokens[cursor + 1].value, line: token.line });
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found && token.value === "import") unknownLines.add(token.line);
|
||||
}
|
||||
return { imports, unknownLines: [...unknownLines].sort((a, b) => a - b) };
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> | undefined {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
|
||||
}
|
||||
|
||||
function objectName(value: unknown): string | undefined {
|
||||
const name = record(record(value)?.metadata)?.name;
|
||||
return typeof name === "string" && name.length > 0 ? name : undefined;
|
||||
}
|
||||
|
||||
function collectWorkloadSecretRefs(podSpecValue: unknown, refs: Set<string>): void {
|
||||
const podSpec = record(podSpecValue);
|
||||
if (!podSpec) return;
|
||||
const addName = (value: unknown): void => {
|
||||
const name = record(value)?.name;
|
||||
if (typeof name === "string" && name.length > 0) refs.add(name);
|
||||
};
|
||||
for (const pullSecret of Array.isArray(podSpec.imagePullSecrets) ? podSpec.imagePullSecrets : []) addName(pullSecret);
|
||||
for (const containerKey of ["initContainers", "containers", "ephemeralContainers"]) {
|
||||
for (const containerValue of Array.isArray(podSpec[containerKey]) ? podSpec[containerKey] as unknown[] : []) {
|
||||
const container = record(containerValue);
|
||||
if (!container) continue;
|
||||
for (const source of Array.isArray(container.envFrom) ? container.envFrom : []) addName(record(source)?.secretRef);
|
||||
for (const environment of Array.isArray(container.env) ? container.env : []) addName(record(record(environment)?.valueFrom)?.secretKeyRef);
|
||||
}
|
||||
}
|
||||
for (const volumeValue of Array.isArray(podSpec.volumes) ? podSpec.volumes : []) {
|
||||
const volume = record(volumeValue);
|
||||
const secretName = record(volume?.secret)?.secretName;
|
||||
if (typeof secretName === "string" && secretName.length > 0) refs.add(secretName);
|
||||
const sources = record(volume?.projected)?.sources;
|
||||
for (const source of Array.isArray(sources) ? sources : []) addName(record(source)?.secret);
|
||||
}
|
||||
}
|
||||
|
||||
function collectSchemaRefs(value: unknown, refs: Set<string>, seen = new WeakSet<object>()): void {
|
||||
if (value === null || typeof value !== "object" || seen.has(value)) return;
|
||||
seen.add(value);
|
||||
if (Array.isArray(value)) { value.forEach((item) => collectSchemaRefs(item, refs, seen)); return; }
|
||||
for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (key === "$ref" && typeof child === "string" && child.startsWith("#/components/schemas/")) refs.add(child.slice("#/components/schemas/".length));
|
||||
else collectSchemaRefs(child, refs, seen);
|
||||
}
|
||||
}
|
||||
|
||||
function validateBoundedInput(request: SourceImportRequest): void {
|
||||
const limit = request.maxBytes ?? 1_048_576;
|
||||
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 8_388_608) throw new Error("maxBytes must be an integer between 1 and 8388608");
|
||||
if (!request.path || request.path.includes("\0")) throw new Error("Source path must be a non-empty safe string");
|
||||
if (request.input.type === "file") return;
|
||||
if (request.input.type === "text") {
|
||||
const size = new TextEncoder().encode(request.input.text).byteLength;
|
||||
if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`);
|
||||
return;
|
||||
}
|
||||
let size = 0;
|
||||
const seen = new WeakSet<object>();
|
||||
const pending: Array<{ value: unknown; depth: number }> = [{ value: request.input.value, depth: 0 }];
|
||||
while (pending.length > 0) {
|
||||
const { value, depth } = pending.pop()!;
|
||||
if (depth > 128) throw new Error("Object input nesting exceeds maximum depth 128");
|
||||
if (value === null || typeof value === "boolean") size += 4;
|
||||
else if (typeof value === "number") {
|
||||
if (!Number.isFinite(value)) throw new Error("Object input must contain finite JSON values");
|
||||
size += 8;
|
||||
} else if (typeof value === "string") size += new TextEncoder().encode(value).byteLength;
|
||||
else {
|
||||
if (typeof value !== "object") throw new Error("Object input must contain plain JSON values");
|
||||
if (seen.has(value)) throw new Error("Object input must contain acyclic plain JSON values");
|
||||
seen.add(value);
|
||||
if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new Error("Object input must contain plain JSON objects");
|
||||
const descriptors = Object.getOwnPropertyDescriptors(value);
|
||||
for (const [key, descriptor] of Object.entries(descriptors)) {
|
||||
if (key === "length") continue;
|
||||
if (!("value" in descriptor)) throw new Error("Object input must contain plain JSON data properties");
|
||||
size += new TextEncoder().encode(key).byteLength;
|
||||
if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`);
|
||||
pending.push({ value: descriptor.value, depth: depth + 1 });
|
||||
}
|
||||
}
|
||||
if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function importInline(request: SourceImportRequest): Promise<SourceImportResult> {
|
||||
validateBoundedInput(request);
|
||||
if (request.sourceKind === "ci" && request.input.type === "object") {
|
||||
const jobs = record(record(request.input.value)?.jobs);
|
||||
if (!jobs || Object.keys(jobs).length === 0) throw new Error("Malformed CI input: jobs must be a non-empty object");
|
||||
const names = Object.keys(jobs).sort(codePointCompare);
|
||||
const nodes: DiagramNode[] = names.map((name) => ({ id: id("command", name), label: name, kind: "command", provenance: { path: request.path } }));
|
||||
const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = [];
|
||||
for (const name of names) {
|
||||
const rawNeeds = record(jobs[name])?.needs;
|
||||
const needs = (Array.isArray(rawNeeds) ? rawNeeds : rawNeeds === undefined ? [] : [rawNeeds]).filter((value): value is string => typeof value === "string").sort(codePointCompare);
|
||||
for (const dependency of needs) {
|
||||
if (!names.includes(dependency)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown CI job: ${dependency}`, path: request.path }); continue; }
|
||||
edges.push({ id: id("edge", dependency, "needs", name, String(edges.length)), source: id("command", dependency), target: id("command", name), kind: "needs", provenance: { path: request.path } });
|
||||
}
|
||||
}
|
||||
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
if (request.sourceKind === "openapi" && request.input.type === "object") {
|
||||
const root = record(request.input.value); const paths = record(root?.paths); const schemas = record(record(root?.components)?.schemas) ?? {};
|
||||
if (!root || !paths) throw new Error("Malformed OpenAPI input: paths must be an object");
|
||||
const nodes: DiagramNode[] = []; const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = [];
|
||||
const methods = new Set(["get", "put", "post", "delete", "patch", "options", "head", "trace"]);
|
||||
for (const path of Object.keys(paths).sort(codePointCompare)) {
|
||||
const pathItem = record(paths[path]) ?? {};
|
||||
for (const method of Object.keys(pathItem).filter((key) => methods.has(key.toLowerCase())).sort(codePointCompare)) {
|
||||
const operationId = id("command", method, path);
|
||||
nodes.push({ id: operationId, label: `${method.toUpperCase()} ${path}`, kind: "command", provenance: { path: request.path } });
|
||||
const refs = new Set<string>(); collectSchemaRefs(pathItem[method], refs);
|
||||
for (const schema of [...refs].sort(codePointCompare)) {
|
||||
if (!Object.prototype.hasOwnProperty.call(schemas, schema)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown schema: ${schema}`, path: request.path }); continue; }
|
||||
edges.push({ id: id("edge", method, path, schema, String(edges.length)), source: operationId, target: id("schema", schema), kind: "schema-reference", provenance: { path: request.path } });
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const schema of Object.keys(schemas).sort(codePointCompare)) nodes.push({ id: id("schema", schema), label: schema, kind: "library", provenance: { path: request.path } });
|
||||
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
if (request.sourceKind === "sql" && request.input.type === "text") {
|
||||
const scanned = maskComments(request.input.text, ["--"], false, "'");
|
||||
const tables = new Map<string, number>(); const refs: Array<{ source: string; target: string; line: number }> = [];
|
||||
const diagnostics: ImportDiagnostic[] = scanned.unknownLines.map((line) => ({ code: "unknown-construct", severity: "warning", message: "SQL construct is outside the bounded static subset", path: request.path, line }));
|
||||
let current: string | undefined;
|
||||
scanned.text.split(/\r?\n/).forEach((line, index) => {
|
||||
const declaration = line.match(/^\s*CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?["`\[]?([A-Za-z_]\w*)/i)?.[1];
|
||||
if (declaration) {
|
||||
if (tables.has(declaration)) diagnostics.push({ code: "duplicate-id", severity: "warning", message: `Duplicate table: ${declaration}`, path: request.path, line: index + 1 });
|
||||
else tables.set(declaration, index + 1);
|
||||
current = declaration;
|
||||
}
|
||||
const target = line.match(/\bREFERENCES\s+["`\[]?([A-Za-z_]\w*)/i)?.[1];
|
||||
if (current && target) refs.push({ source: current, target, line: index + 1 });
|
||||
if (/\)\s*;/.test(line)) current = undefined;
|
||||
});
|
||||
const nodes: DiagramNode[] = [...tables].sort(([a], [b]) => codePointCompare(a, b)).map(([name, line]) => ({ id: id("table", name), label: name, kind: "table", provenance: { path: request.path, line } }));
|
||||
const edges: DiagramEdge[] = refs.filter((ref) => tables.has(ref.target)).map((ref, occurrence) => ({ id: id("edge", ref.source, "fk", ref.target, String(occurrence)), source: id("table", ref.source), target: id("table", ref.target), kind: "foreign-key", provenance: { path: request.path, line: ref.line } }));
|
||||
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
if (request.sourceKind === "docker-compose" && request.input.type === "object") {
|
||||
const root = record(request.input.value);
|
||||
const services = record(root?.services);
|
||||
const volumes = record(root?.volumes) ?? {};
|
||||
if (!services || Object.keys(services).length === 0) throw new Error("Malformed Docker Compose input: services must be a non-empty object");
|
||||
const serviceNames = Object.keys(services).sort(codePointCompare); const volumeNames = Object.keys(volumes).sort(codePointCompare);
|
||||
const nodes: DiagramNode[] = [
|
||||
...serviceNames.map((name) => ({ id: id("service", name), label: name, kind: "service", provenance: { path: request.path } })),
|
||||
...volumeNames.map((name) => ({ id: id("volume", name), label: name, kind: "volume", provenance: { path: request.path } })),
|
||||
];
|
||||
const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = [];
|
||||
for (const name of serviceNames) {
|
||||
const service = record(services[name]) ?? {};
|
||||
const depends = Array.isArray(service.depends_on) ? service.depends_on : Object.keys(record(service.depends_on) ?? {});
|
||||
for (const dependency of depends.filter((value): value is string => typeof value === "string").sort(codePointCompare)) {
|
||||
if (!serviceNames.includes(dependency)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown service: ${dependency}`, path: request.path }); continue; }
|
||||
edges.push({ id: id("edge", name, "depends", dependency, String(edges.length)), source: id("service", name), target: id("service", dependency), kind: "depends-on", provenance: { path: request.path } });
|
||||
}
|
||||
for (const mount of (Array.isArray(service.volumes) ? service.volumes : []).filter((value): value is string => typeof value === "string").sort(codePointCompare)) {
|
||||
const volume = mount.split(":", 1)[0];
|
||||
if (volumeNames.includes(volume)) edges.push({ id: id("edge", name, "mounts", volume, String(edges.length)), source: id("service", name), target: id("volume", volume), kind: "mounts", provenance: { path: request.path } });
|
||||
}
|
||||
}
|
||||
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
if (request.sourceKind === "kubernetes" && request.input.type === "object") {
|
||||
const values = Array.isArray(request.input.value) ? request.input.value : [request.input.value];
|
||||
const rawObjects = values.map(record).filter((value): value is Record<string, unknown> => Boolean(value && typeof value.kind === "string" && objectName(value)));
|
||||
const occurrences = new Map<string, number>();
|
||||
const objects = rawObjects.map((value) => {
|
||||
const kind = String(value.kind);
|
||||
const name = objectName(value)!;
|
||||
const rawNamespace = record(value.metadata)?.namespace;
|
||||
const namespace = typeof rawNamespace === "string" && rawNamespace.length > 0 ? rawNamespace : "default";
|
||||
const key = JSON.stringify([kind, namespace, name]);
|
||||
const occurrence = occurrences.get(key) ?? 0;
|
||||
occurrences.set(key, occurrence + 1);
|
||||
return { value, kind, name, namespace, nodeId: id("k8s", kind, namespace, name, String(occurrence)) };
|
||||
});
|
||||
const diagnostics: ImportDiagnostic[] = objects.filter((object) => object.kind === "Secret").map(() => ({ code: "secret-redacted", severity: "warning", message: "Kubernetes Secret payload omitted", path: request.path }));
|
||||
const nodes: DiagramNode[] = objects.map((object) => ({
|
||||
id: object.nodeId,
|
||||
label: `${object.kind}/${object.namespace}/${object.name}`,
|
||||
kind: object.kind.toLowerCase(),
|
||||
provenance: { path: request.path },
|
||||
})).sort((a, b) => codePointCompare(a.label, b.label));
|
||||
const byKey = new Map<string, typeof objects[number]>();
|
||||
for (const object of objects) {
|
||||
const key = JSON.stringify([object.kind, object.namespace, object.name]);
|
||||
if (!byKey.has(key)) byKey.set(key, object);
|
||||
}
|
||||
const workloadKinds = new Set(["Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "Job"]);
|
||||
const edges: DiagramEdge[] = [];
|
||||
for (const object of objects) {
|
||||
const secretRefs = new Set<string>();
|
||||
if (workloadKinds.has(object.kind)) collectWorkloadSecretRefs(record(record(object.value.spec)?.template)?.spec, secretRefs);
|
||||
for (const target of [...secretRefs].sort(codePointCompare)) {
|
||||
const secret = byKey.get(JSON.stringify(["Secret", object.namespace, target]));
|
||||
if (secret) edges.push({ id: id("edge", object.nodeId, "secret", secret.nodeId, String(edges.length)), source: object.nodeId, target: secret.nodeId, kind: "uses-secret", provenance: { path: request.path } });
|
||||
}
|
||||
if (object.kind === "Service") {
|
||||
const selector = record(record(object.value.spec)?.selector);
|
||||
if (!selector || Object.keys(selector).length === 0) continue;
|
||||
for (const workload of objects.filter((candidate) => workloadKinds.has(candidate.kind) && candidate.namespace === object.namespace)) {
|
||||
const labels = record(record(record(record(workload.value.spec)?.template)?.metadata)?.labels);
|
||||
if (labels && Object.entries(selector).every(([key, selected]) => labels[key] === selected)) {
|
||||
edges.push({ id: id("edge", object.nodeId, "selects", workload.nodeId, String(edges.length)), source: object.nodeId, target: workload.nodeId, kind: "selects", provenance: { path: request.path } });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
if (request.sourceKind === "terraform" && request.input.type === "text") {
|
||||
const heredocs = maskTerraformHeredocs(request.input.text);
|
||||
const structural = maskComments(heredocs.text, ["#", "//"]);
|
||||
const referencesOnly = maskTerraformStrings(structural.text);
|
||||
const unknownLines = [...new Set([...heredocs.unknownLines, ...structural.unknownLines, ...referencesOnly.unknownLines])].sort((a, b) => a - b);
|
||||
const diagnostics: ImportDiagnostic[] = unknownLines.map((line) => ({ code: "unknown-construct", severity: "warning", message: "Terraform construct is outside the bounded static subset", path: request.path, line }));
|
||||
const resources = new Map<string, number>();
|
||||
const references: Array<{ source: string; target: string; line: number }> = [];
|
||||
const referenceLines = referencesOnly.text.split(/\r?\n/);
|
||||
let current: string | undefined;
|
||||
structural.text.split(/\r?\n/).forEach((line, index) => {
|
||||
const declaration = line.match(/^\s*resource\s+"([^"]+)"\s+"([^"]+)"\s*\{/);
|
||||
if (declaration) {
|
||||
current = `${declaration[1]}.${declaration[2]}`;
|
||||
resources.set(current, index + 1);
|
||||
if (/\{\s*\}/.test(line)) current = undefined;
|
||||
return;
|
||||
}
|
||||
if (current) {
|
||||
for (const match of referenceLines[index].matchAll(/\b([A-Za-z_]\w*\.[A-Za-z_]\w*)\.[A-Za-z_]\w*/g)) references.push({ source: current, target: match[1], line: index + 1 });
|
||||
if (/^\s*}/.test(line)) current = undefined;
|
||||
}
|
||||
});
|
||||
const nodes = [...resources].sort(([a], [b]) => codePointCompare(a, b)).map(([name, line]) => ({ id: id("resource", name), label: name, kind: "resource", provenance: { path: request.path, line } }));
|
||||
const edges = references.filter((ref) => resources.has(ref.target)).sort((a, b) => codePointCompare(a.source, b.source) || codePointCompare(a.target, b.target) || a.line - b.line).map((ref, occurrence) => ({ id: id("edge", ref.source, ref.target, String(occurrence)), source: id("resource", ref.source), target: id("resource", ref.target), kind: "reference", provenance: { path: request.path, line: ref.line } }));
|
||||
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
if (!(["python", "javascript", "typescript", "go", "rust"] as SourceKind[]).includes(request.sourceKind) || request.input.type !== "text") throw new Error("Unsupported source input");
|
||||
const nodes: DiagramNode[] = [{ id: id("module", request.path), label: request.path, kind: "module", provenance: { path: request.path, line: 1 } }];
|
||||
const edges: DiagramEdge[] = [];
|
||||
const diagnostics: ImportDiagnostic[] = [];
|
||||
const javaScriptScan = request.sourceKind === "javascript" || request.sourceKind === "typescript" ? scanJavaScriptImports(request.input.text) : undefined;
|
||||
const nonJavaScriptScan = request.sourceKind === "python" ? maskPythonTripleStrings(request.input.text)
|
||||
: request.sourceKind === "go" ? maskComments(request.input.text, ["//"], false, "`")
|
||||
: request.sourceKind === "rust" ? maskComments(request.input.text, ["//"], true, "", true)
|
||||
: { text: request.input.text, unknownLines: [] };
|
||||
for (const line of javaScriptScan?.unknownLines ?? nonJavaScriptScan.unknownLines) diagnostics.push({ code: "unknown-construct", severity: "warning", message: `${request.sourceKind} construct is outside the bounded static-import subset`, path: request.path, line });
|
||||
nonJavaScriptScan.text.split(/\r?\n/).forEach((line, index) => {
|
||||
const dynamicPython = request.sourceKind === "python" && /\b(?:__import__|importlib\.import_module)\s*\(\s*[^"']/.test(line);
|
||||
if (dynamicPython) diagnostics.push({ code: "unknown-construct", severity: "warning", message: "Dynamic imports are not evaluated", path: request.path, line: index + 1 });
|
||||
});
|
||||
const seen = new Set<string>();
|
||||
const foundImports = request.sourceKind === "rust"
|
||||
? nonJavaScriptScan.text.split(/\r?\n/).flatMap((line, index) => {
|
||||
const mod = line.match(/^\s*mod\s+([A-Za-z_]\w*)\s*;/)?.[1];
|
||||
const use = line.match(/^\s*use\s+([^;]+);/)?.[1];
|
||||
if (/\w+!\s*\(/.test(line)) diagnostics.push({ code: "unknown-construct", severity: "warning", message: "Rust macros are not expanded", path: request.path, line: index + 1 });
|
||||
if (mod) return [{ name: mod, line: index + 1 }];
|
||||
if (use) {
|
||||
const root = use.includes("::{") ? use.slice(0, use.indexOf("::{")) : use.split("::").slice(0, -1).join("::") || use;
|
||||
return [{ name: root, line: index + 1 }];
|
||||
}
|
||||
return [];
|
||||
})
|
||||
: javaScriptScan?.imports ?? codeImports(request.sourceKind as "python" | "go", nonJavaScriptScan.text);
|
||||
for (const found of foundImports) {
|
||||
if (seen.has(found.name)) continue;
|
||||
seen.add(found.name);
|
||||
const targetId = id("library", found.name);
|
||||
nodes.push({ id: targetId, label: found.name, kind: "library", provenance: { path: request.path, line: found.line } });
|
||||
edges.push({ id: id("edge", request.path, found.name, String(edges.length)), source: nodes[0].id, target: targetId, kind: "import", provenance: { path: request.path, line: found.line } });
|
||||
}
|
||||
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
|
||||
function isConfined(root: string, candidate: string): boolean {
|
||||
const pathFromRoot = relative(root, candidate);
|
||||
return pathFromRoot === "" || (!pathFromRoot.startsWith("..") && !isAbsolute(pathFromRoot));
|
||||
}
|
||||
|
||||
async function openConfined(candidate: string, root: string): Promise<{ handle: FileHandle; canonical: string }> {
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await open(candidate, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ELOOP") throw new Error("Source symlink escape risk cannot be opened safely");
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const canonical = await realpath(`/proc/self/fd/${handle.fd}`);
|
||||
if (!isConfined(root, canonical)) throw new Error("Opened source escapes outside declared root");
|
||||
return { handle, canonical };
|
||||
} catch (error) {
|
||||
await handle.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function readBounded(handle: FileHandle, limit: number, directory: boolean): Promise<Buffer> {
|
||||
const initial = await handle.stat();
|
||||
if (!initial.isFile()) throw new Error("Source entry must be a regular file");
|
||||
if (initial.size > limit) throw new Error(`${directory ? "Directory source input" : "Source input"} exceeds maxBytes ${limit}`);
|
||||
const chunks: Buffer[] = [];
|
||||
let consumed = 0;
|
||||
while (consumed < limit) {
|
||||
const chunk = Buffer.allocUnsafe(Math.min(65_536, limit - consumed));
|
||||
const { bytesRead } = await handle.read(chunk, 0, chunk.byteLength, null);
|
||||
if (bytesRead === 0) break;
|
||||
chunks.push(chunk.subarray(0, bytesRead));
|
||||
consumed += bytesRead;
|
||||
}
|
||||
const final = await handle.stat();
|
||||
if (initial.dev !== final.dev || initial.ino !== final.ino) throw new Error("Opened source identity changed during read");
|
||||
if (final.size > limit || consumed < initial.size) throw new Error(`${directory ? "Directory source input" : "Source input"} exceeds maxBytes ${limit}`);
|
||||
return Buffer.concat(chunks, consumed);
|
||||
}
|
||||
|
||||
export async function importSource(request: SourceImportRequest): Promise<SourceImportResult> {
|
||||
validateBoundedInput(request);
|
||||
if (request.input.type !== "file") return importInline(request);
|
||||
if (isAbsolute(request.path)) throw new Error("Source path escapes outside declared root");
|
||||
const rootPath = await realpath(request.input.root);
|
||||
const lexicalPath = resolve(rootPath, request.path);
|
||||
if (!isConfined(rootPath, lexicalPath)) throw new Error("Source path escapes outside declared root");
|
||||
const limit = request.maxBytes ?? 1_048_576;
|
||||
const objectKinds = new Set<SourceKind>(["kubernetes", "docker-compose", "openapi", "ci"]);
|
||||
const extensions: Record<SourceKind, Set<string>> = {
|
||||
python: new Set([".py"]), javascript: new Set([".js", ".jsx", ".mjs", ".cjs"]), typescript: new Set([".ts", ".tsx", ".mts", ".cts"]),
|
||||
go: new Set([".go"]), rust: new Set([".rs"]), terraform: new Set([".tf"]), sql: new Set([".sql"]),
|
||||
kubernetes: new Set([".json", ".yaml", ".yml"]), "docker-compose": new Set([".json", ".yaml", ".yml"]), openapi: new Set([".json", ".yaml", ".yml"]), ci: new Set([".json", ".yaml", ".yml"]),
|
||||
};
|
||||
const parse = async (buffer: Buffer, provenancePath: string): Promise<SourceImportResult> => {
|
||||
const text = new TextDecoder("utf-8", { fatal: true }).decode(buffer);
|
||||
let input: SourceInput;
|
||||
if (objectKinds.has(request.sourceKind)) {
|
||||
const yaml = /\.ya?ml$/i.test(provenancePath);
|
||||
try {
|
||||
if (yaml) {
|
||||
const documents = loadAll(text, undefined, { schema: JSON_SCHEMA, json: false }).filter((value) => value !== undefined);
|
||||
if (documents.length === 0) throw new Error("empty");
|
||||
if (request.sourceKind !== "kubernetes" && documents.length !== 1) throw new Error("multiple documents");
|
||||
input = { type: "object", value: request.sourceKind === "kubernetes" ? documents : documents[0] };
|
||||
} else input = { type: "object", value: JSON.parse(text) as unknown };
|
||||
} catch { throw new Error(`Malformed ${yaml ? "YAML or disallowed tag" : "JSON"} source: ${provenancePath}`); }
|
||||
} else input = { type: "text", text };
|
||||
return importInline({ ...request, path: provenancePath, input });
|
||||
};
|
||||
const source = await openConfined(lexicalPath, rootPath);
|
||||
const info = await source.handle.stat();
|
||||
if (info.isFile()) {
|
||||
try { return await parse(await readBounded(source.handle, limit, false), request.path); }
|
||||
finally { await source.handle.close(); }
|
||||
}
|
||||
if (!info.isDirectory()) { await source.handle.close(); throw new Error("Source path must be a file or directory"); }
|
||||
|
||||
const results: SourceImportResult[] = [];
|
||||
let consumed = 0;
|
||||
const walk = async (directory: FileHandle): Promise<void> => {
|
||||
for (const name of (await readdir(`/proc/self/fd/${directory.fd}`)).sort(codePointCompare)) {
|
||||
const opened = await openConfined(resolve(`/proc/self/fd/${directory.fd}`, name), rootPath);
|
||||
try {
|
||||
const entryInfo = await opened.handle.stat();
|
||||
if (entryInfo.isDirectory()) await walk(opened.handle);
|
||||
else if (entryInfo.isFile()) {
|
||||
const dot = name.lastIndexOf("."); const extension = dot >= 0 ? name.slice(dot).toLowerCase() : "";
|
||||
if (extensions[request.sourceKind].has(extension)) {
|
||||
const provenancePath = relative(rootPath, opened.canonical).split("\\").join("/");
|
||||
const buffer = await readBounded(opened.handle, limit - consumed, true);
|
||||
consumed += buffer.byteLength;
|
||||
results.push(await parse(buffer, provenancePath));
|
||||
}
|
||||
} else throw new Error("Source directory contains an unsupported entry type");
|
||||
} finally { await opened.handle.close(); }
|
||||
}
|
||||
};
|
||||
try { await walk(source.handle); }
|
||||
finally { await source.handle.close(); }
|
||||
if (results.length === 0) throw new Error(`No supported ${request.sourceKind} source files found`);
|
||||
const nodeById = new Map<string, DiagramNode>(); const edgeById = new Map<string, DiagramEdge>(); const diagnostics: ImportDiagnostic[] = [];
|
||||
for (const result of results) {
|
||||
diagnostics.push(...result.diagnostics);
|
||||
for (const node of result.diagram.pages[0].nodes) if (!nodeById.has(node.id)) nodeById.set(node.id, node);
|
||||
for (const edge of result.diagram.pages[0].edges) {
|
||||
if (edgeById.has(edge.id)) diagnostics.push({ code: "duplicate-id", severity: "warning", message: `Duplicate edge omitted: ${edge.id}`, path: String(edge.provenance?.path ?? request.path) });
|
||||
else edgeById.set(edge.id, edge);
|
||||
}
|
||||
}
|
||||
return { diagram: diagramFor(request, [...nodeById.values()], [...edgeById.values()]), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
||||
}
|
||||
Reference in New Issue
Block a user