746 lines
42 KiB
TypeScript
746 lines
42 KiB
TypeScript
import { constants } from "node:fs";
|
|
import { open, readdir, realpath } from "node:fs/promises";
|
|
import type { FileHandle } from "node:fs/promises";
|
|
import { isAbsolute, relative, resolve } from "node:path";
|
|
|
|
import { JSON_SCHEMA, loadAll } from "js-yaml";
|
|
|
|
import type { DiagramEdge, DiagramIRV2, DiagramNode } from "../../model/diagram-ir.js";
|
|
import { validateDiagramIR } from "../../model/diagram-ir.js";
|
|
|
|
export type SourceKind = "python" | "javascript" | "typescript" | "go" | "rust" | "terraform" | "kubernetes" | "docker-compose" | "sql" | "openapi" | "ci";
|
|
export type SourceInput = { type: "text"; text: string } | { type: "object"; value: unknown } | { type: "file"; root: string };
|
|
export interface SourceImportRequest { sourceKind: SourceKind; path: string; input: SourceInput; maxBytes?: number }
|
|
export interface ImportDiagnostic { code: string; severity: "warning" | "error"; message: string; path: string; line?: number }
|
|
export interface SourceImportResult { diagram: DiagramIRV2; diagnostics: ImportDiagnostic[]; provenance: { sourceKind: SourceKind; path: string } }
|
|
|
|
function id(prefix: string, ...values: string[]): string {
|
|
const encoded = values.map((value) => Buffer.from(value, "utf8").toString("hex") || "0");
|
|
return `${prefix}-${encoded.join(".")}`;
|
|
}
|
|
|
|
function codePointCompare(left: string, right: string): number {
|
|
const leftPoints = [...left];
|
|
const rightPoints = [...right];
|
|
const length = Math.min(leftPoints.length, rightPoints.length);
|
|
for (let index = 0; index < length; index += 1) {
|
|
const difference = leftPoints[index].codePointAt(0)! - rightPoints[index].codePointAt(0)!;
|
|
if (difference !== 0) return difference;
|
|
}
|
|
return leftPoints.length - rightPoints.length;
|
|
}
|
|
|
|
function diagramFor(request: SourceImportRequest, nodes: DiagramNode[], edges: DiagramEdge[]): DiagramIRV2 {
|
|
return validateDiagramIR({ version: 2, title: request.path, pages: [{ id: "source-import", title: request.path, nodes, edges, layout: { type: "layered", direction: "horizontal" } }], provenance: { sourceKind: request.sourceKind, path: request.path } }) as DiagramIRV2;
|
|
}
|
|
|
|
function codeImports(kind: "python" | "go", text: string): Array<{ name: string; line: number }> {
|
|
const imports: Array<{ name: string; line: number }> = [];
|
|
let goBlock = false;
|
|
text.split(/\r?\n/).forEach((line, index) => {
|
|
let name: string | undefined;
|
|
if (kind === "python") {
|
|
const match = line.match(/^\s*(?:import\s+([A-Za-z_][\w.]*)|from\s+([A-Za-z_][\w.]*)\s+import\s+)/);
|
|
name = match?.[1] ?? match?.[2];
|
|
} else if (kind === "go") {
|
|
if (/^\s*import\s*\(\s*$/.test(line)) { goBlock = true; return; }
|
|
if (goBlock && /^\s*\)/.test(line)) { goBlock = false; return; }
|
|
const match = goBlock ? line.match(/^\s*(?:[._A-Za-z]\w*\s+)?"([^"]+)"/) : line.match(/^\s*import\s+(?:[._A-Za-z]\w*\s+)?"([^"]+)"/);
|
|
name = match?.[1];
|
|
}
|
|
if (name) imports.push({ name, line: index + 1 });
|
|
});
|
|
return imports;
|
|
}
|
|
|
|
interface MaskedSource { text: string; unknownLines: number[] }
|
|
|
|
function maskRange(characters: string[], start: number, end: number): void {
|
|
for (let index = start; index < end; index += 1) if (characters[index] !== "\n" && characters[index] !== "\r") characters[index] = " ";
|
|
}
|
|
|
|
function maskPythonTripleStrings(text: string): MaskedSource {
|
|
const characters = text.split("");
|
|
const unknownLines: number[] = [];
|
|
let offset = 0; let line = 1;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
|
const quote = text.startsWith("'''", offset) ? "'''" : text.startsWith('"""', offset) ? '"""' : undefined;
|
|
if (!quote) { offset += 1; continue; }
|
|
const start = offset; const startLine = line;
|
|
offset += 3;
|
|
let closed = false;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
|
if (text[offset] === "\n") line += 1;
|
|
if (text.startsWith(quote, offset)) { offset += 3; closed = true; break; }
|
|
offset += 1;
|
|
}
|
|
maskRange(characters, start, offset);
|
|
if (!closed) unknownLines.push(startLine);
|
|
}
|
|
return { text: characters.join(""), unknownLines };
|
|
}
|
|
|
|
function maskComments(text: string, lineMarkers: string[], nestedBlock = false, maskedQuotes = "", rustRawStrings = false): MaskedSource {
|
|
const characters = text.split("");
|
|
const unknownLines: number[] = [];
|
|
let offset = 0; let line = 1;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
|
const marker = lineMarkers.find((candidate) => text.startsWith(candidate, offset));
|
|
if (marker) {
|
|
const start = offset;
|
|
while (offset < text.length && text[offset] !== "\n") offset += 1;
|
|
maskRange(characters, start, offset);
|
|
continue;
|
|
}
|
|
if (text.startsWith("/*", offset)) {
|
|
const start = offset; const startLine = line;
|
|
offset += 2;
|
|
let depth = 1;
|
|
while (offset < text.length && depth > 0) {
|
|
if (text[offset] === "\n") line += 1;
|
|
if (nestedBlock && text.startsWith("/*", offset)) { depth += 1; offset += 2; continue; }
|
|
if (text.startsWith("*/", offset)) { depth -= 1; offset += 2; continue; }
|
|
offset += 1;
|
|
}
|
|
maskRange(characters, start, offset);
|
|
if (depth > 0) unknownLines.push(startLine);
|
|
continue;
|
|
}
|
|
if (rustRawStrings && text[offset] === "r") {
|
|
let delimiterEnd = offset + 1;
|
|
while (text[delimiterEnd] === "#") delimiterEnd += 1;
|
|
if (text[delimiterEnd] === '"') {
|
|
const start = offset; const startLine = line;
|
|
const terminator = `"${"#".repeat(delimiterEnd - offset - 1)}`;
|
|
offset = delimiterEnd + 1;
|
|
const rawEnd = text.indexOf(terminator, offset);
|
|
const closed = rawEnd >= 0;
|
|
const next = closed ? rawEnd + terminator.length : text.length;
|
|
line += (text.slice(offset, next).match(/\n/g) ?? []).length;
|
|
offset = next;
|
|
maskRange(characters, start, offset);
|
|
if (!closed) unknownLines.push(startLine);
|
|
continue;
|
|
}
|
|
}
|
|
if (text[offset] === "'" || text[offset] === '"' || text[offset] === "`") {
|
|
const quote = text[offset]; const start = offset; const startLine = line;
|
|
offset += 1;
|
|
let closed = false;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\\" && quote !== "`" && offset + 1 < text.length) { offset += 2; continue; }
|
|
if (quote === "'" && text[offset] === "'" && text[offset + 1] === "'") { offset += 2; continue; }
|
|
if (text[offset] === "\n") line += 1;
|
|
if (text[offset] === quote) { offset += 1; closed = true; break; }
|
|
offset += 1;
|
|
}
|
|
if (maskedQuotes.includes(quote)) maskRange(characters, start, offset);
|
|
if (!closed) unknownLines.push(startLine);
|
|
continue;
|
|
}
|
|
offset += 1;
|
|
}
|
|
return { text: characters.join(""), unknownLines };
|
|
}
|
|
|
|
function maskTerraformHeredocs(text: string): MaskedSource {
|
|
const characters = text.split("");
|
|
const unknownLines: number[] = [];
|
|
const heredocOpening = /<<(-?)([A-Za-z_]\w*)[ \t]*(?:\r?\n)/y;
|
|
let offset = 0; let line = 1;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
|
if (text[offset] === "#" || text.startsWith("//", offset)) {
|
|
while (offset < text.length && text[offset] !== "\n") offset += 1;
|
|
continue;
|
|
}
|
|
if (text.startsWith("/*", offset)) {
|
|
const end = text.indexOf("*/", offset + 2);
|
|
const next = end < 0 ? text.length : end + 2;
|
|
line += (text.slice(offset, next).match(/\n/g) ?? []).length;
|
|
offset = next;
|
|
continue;
|
|
}
|
|
if (text[offset] === '"') {
|
|
offset += 1;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
|
if (text[offset] === "\n") line += 1;
|
|
if (text[offset] === '"') { offset += 1; break; }
|
|
offset += 1;
|
|
}
|
|
continue;
|
|
}
|
|
heredocOpening.lastIndex = offset;
|
|
const opening = heredocOpening.exec(text);
|
|
if (!opening) { offset += 1; continue; }
|
|
const start = offset; const startLine = line;
|
|
const indented = opening[1] === "-";
|
|
const delimiter = opening[2];
|
|
offset += opening[0].length;
|
|
line += 1;
|
|
let closed = false;
|
|
while (offset < text.length) {
|
|
const newline = text.indexOf("\n", offset);
|
|
const end = newline < 0 ? text.length : newline;
|
|
const candidate = text.slice(offset, end).replace(/\r$/, "");
|
|
const expected = indented ? candidate.trim() : candidate.trimEnd();
|
|
if (expected === delimiter && (indented || candidate === expected)) {
|
|
offset = end;
|
|
closed = true;
|
|
break;
|
|
}
|
|
offset = newline < 0 ? text.length : newline + 1;
|
|
if (newline >= 0) line += 1;
|
|
}
|
|
maskRange(characters, start, offset);
|
|
if (!closed) unknownLines.push(startLine);
|
|
}
|
|
return { text: characters.join(""), unknownLines };
|
|
}
|
|
|
|
function maskTerraformStrings(text: string): MaskedSource {
|
|
const characters = text.split("");
|
|
const unknownLines: number[] = [];
|
|
let offset = 0; let line = 1;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\n") { line += 1; offset += 1; continue; }
|
|
if (text[offset] !== '"') { offset += 1; continue; }
|
|
const start = offset; const startLine = line;
|
|
offset += 1;
|
|
let closed = false;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
|
if (text[offset] === "\n") line += 1;
|
|
if (text[offset] === '"') { offset += 1; closed = true; break; }
|
|
offset += 1;
|
|
}
|
|
maskRange(characters, start, offset);
|
|
if (!closed) unknownLines.push(startLine);
|
|
}
|
|
return { text: characters.join(""), unknownLines };
|
|
}
|
|
|
|
interface JavaScriptToken { kind: "identifier" | "string" | "punctuation"; value: string; line: number }
|
|
|
|
function scanJavaScriptImports(text: string): { imports: Array<{ name: string; line: number }>; unknownLines: number[] } {
|
|
const tokens: JavaScriptToken[] = [];
|
|
const unknownLines = new Set<number>();
|
|
let offset = 0;
|
|
let line = 1;
|
|
while (offset < text.length) {
|
|
const character = text[offset];
|
|
if (character === "\n") { line += 1; offset += 1; continue; }
|
|
if (/\s/.test(character)) { offset += 1; continue; }
|
|
if (character === "/" && text[offset + 1] === "/") {
|
|
offset += 2;
|
|
while (offset < text.length && text[offset] !== "\n") offset += 1;
|
|
continue;
|
|
}
|
|
if (character === "/" && text[offset + 1] === "*") {
|
|
const startLine = line;
|
|
offset += 2;
|
|
let closed = false;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\n") line += 1;
|
|
if (text[offset] === "*" && text[offset + 1] === "/") { offset += 2; closed = true; break; }
|
|
offset += 1;
|
|
}
|
|
if (!closed) unknownLines.add(startLine);
|
|
continue;
|
|
}
|
|
if (character === "/") {
|
|
const previous = tokens[tokens.length - 1];
|
|
const followsArrow = previous?.value === ">" && tokens[tokens.length - 2]?.value === "=";
|
|
const regexPrefix = !previous || followsArrow || (previous.kind === "punctuation" && "=([{,:;!&|?".includes(previous.value)) || (previous.kind === "identifier" && new Set(["return", "case", "throw", "typeof", "instanceof", "in", "of", "yield", "await"]).has(previous.value));
|
|
if (regexPrefix) {
|
|
const startLine = line;
|
|
let inCharacterClass = false;
|
|
let closed = false;
|
|
offset += 1;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
|
if (text[offset] === "\n") break;
|
|
if (text[offset] === "[") inCharacterClass = true;
|
|
else if (text[offset] === "]") inCharacterClass = false;
|
|
else if (text[offset] === "/" && !inCharacterClass) { offset += 1; closed = true; break; }
|
|
offset += 1;
|
|
}
|
|
while (closed && offset < text.length && /[A-Za-z]/.test(text[offset])) offset += 1;
|
|
if (!closed) unknownLines.add(startLine);
|
|
continue;
|
|
}
|
|
}
|
|
if (character === "'" || character === '"') {
|
|
const quote = character;
|
|
const startLine = line;
|
|
let value = "";
|
|
offset += 1;
|
|
let closed = false;
|
|
while (offset < text.length) {
|
|
const child = text[offset];
|
|
if (child === "\\" && offset + 1 < text.length) { value += child + text[offset + 1]; offset += 2; continue; }
|
|
if (child === quote) { offset += 1; closed = true; break; }
|
|
if (child === "\n") line += 1;
|
|
value += child;
|
|
offset += 1;
|
|
}
|
|
if (closed) tokens.push({ kind: "string", value, line: startLine });
|
|
else unknownLines.add(startLine);
|
|
continue;
|
|
}
|
|
if (character === "`") {
|
|
const startLine = line;
|
|
let closed = false;
|
|
let hasExpression = false;
|
|
offset += 1;
|
|
while (offset < text.length) {
|
|
if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; }
|
|
if (text[offset] === "$" && text[offset + 1] === "{") hasExpression = true;
|
|
if (text[offset] === "\n") line += 1;
|
|
if (text[offset] === "`") { offset += 1; closed = true; break; }
|
|
offset += 1;
|
|
}
|
|
if (hasExpression || !closed) unknownLines.add(startLine);
|
|
continue;
|
|
}
|
|
if (/[A-Za-z_$]/.test(character)) {
|
|
const start = offset;
|
|
offset += 1;
|
|
while (offset < text.length && /[A-Za-z0-9_$]/.test(text[offset])) offset += 1;
|
|
tokens.push({ kind: "identifier", value: text.slice(start, offset), line });
|
|
continue;
|
|
}
|
|
tokens.push({ kind: "punctuation", value: character, line });
|
|
offset += 1;
|
|
}
|
|
|
|
const imports: Array<{ name: string; line: number }> = [];
|
|
for (let index = 0; index < tokens.length; index += 1) {
|
|
const token = tokens[index];
|
|
if (token.kind !== "identifier") continue;
|
|
if (token.value === "require" && tokens[index - 1]?.value !== "." && tokens[index + 1]?.value === "(") {
|
|
const argument = tokens[index + 2];
|
|
if (argument?.kind === "string" && tokens[index + 3]?.value === ")") imports.push({ name: argument.value, line: token.line });
|
|
else unknownLines.add(token.line);
|
|
continue;
|
|
}
|
|
if (token.value !== "import" && token.value !== "export") continue;
|
|
if (tokens[index - 1]?.value === ".") continue;
|
|
const next = tokens[index + 1];
|
|
if (token.value === "import" && next?.value === "(") { unknownLines.add(token.line); continue; }
|
|
const previous = tokens[index - 1];
|
|
const statementPosition = !previous || previous.value === ";" || previous.value === "}" || previous.line < token.line;
|
|
if (!statementPosition) continue;
|
|
if (token.value === "import" && next?.kind === "string") { imports.push({ name: next.value, line: token.line }); continue; }
|
|
let found = false;
|
|
for (let cursor = index + 1; cursor < tokens.length && cursor <= index + 256; cursor += 1) {
|
|
if (tokens[cursor].value === ";") break;
|
|
if (tokens[cursor].value === "from" && tokens[cursor + 1]?.kind === "string") {
|
|
imports.push({ name: tokens[cursor + 1].value, line: token.line });
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!found && token.value === "import") unknownLines.add(token.line);
|
|
}
|
|
return { imports, unknownLines: [...unknownLines].sort((a, b) => a - b) };
|
|
}
|
|
|
|
function record(value: unknown): Record<string, unknown> | undefined {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
|
|
}
|
|
|
|
function objectName(value: unknown): string | undefined {
|
|
const name = record(record(value)?.metadata)?.name;
|
|
return typeof name === "string" && name.length > 0 ? name : undefined;
|
|
}
|
|
|
|
function collectWorkloadSecretRefs(podSpecValue: unknown, refs: Set<string>): void {
|
|
const podSpec = record(podSpecValue);
|
|
if (!podSpec) return;
|
|
const addName = (value: unknown): void => {
|
|
const name = record(value)?.name;
|
|
if (typeof name === "string" && name.length > 0) refs.add(name);
|
|
};
|
|
for (const pullSecret of Array.isArray(podSpec.imagePullSecrets) ? podSpec.imagePullSecrets : []) addName(pullSecret);
|
|
for (const containerKey of ["initContainers", "containers", "ephemeralContainers"]) {
|
|
for (const containerValue of Array.isArray(podSpec[containerKey]) ? podSpec[containerKey] as unknown[] : []) {
|
|
const container = record(containerValue);
|
|
if (!container) continue;
|
|
for (const source of Array.isArray(container.envFrom) ? container.envFrom : []) addName(record(source)?.secretRef);
|
|
for (const environment of Array.isArray(container.env) ? container.env : []) addName(record(record(environment)?.valueFrom)?.secretKeyRef);
|
|
}
|
|
}
|
|
for (const volumeValue of Array.isArray(podSpec.volumes) ? podSpec.volumes : []) {
|
|
const volume = record(volumeValue);
|
|
const secretName = record(volume?.secret)?.secretName;
|
|
if (typeof secretName === "string" && secretName.length > 0) refs.add(secretName);
|
|
const sources = record(volume?.projected)?.sources;
|
|
for (const source of Array.isArray(sources) ? sources : []) addName(record(source)?.secret);
|
|
}
|
|
}
|
|
|
|
function collectSchemaRefs(value: unknown, refs: Set<string>, seen = new WeakSet<object>()): void {
|
|
if (value === null || typeof value !== "object" || seen.has(value)) return;
|
|
seen.add(value);
|
|
if (Array.isArray(value)) { value.forEach((item) => collectSchemaRefs(item, refs, seen)); return; }
|
|
for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
|
|
if (key === "$ref" && typeof child === "string" && child.startsWith("#/components/schemas/")) refs.add(child.slice("#/components/schemas/".length));
|
|
else collectSchemaRefs(child, refs, seen);
|
|
}
|
|
}
|
|
|
|
function validateBoundedInput(request: SourceImportRequest): void {
|
|
const limit = request.maxBytes ?? 1_048_576;
|
|
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 8_388_608) throw new Error("maxBytes must be an integer between 1 and 8388608");
|
|
if (!request.path || request.path.includes("\0")) throw new Error("Source path must be a non-empty safe string");
|
|
if (request.input.type === "file") return;
|
|
if (request.input.type === "text") {
|
|
const size = new TextEncoder().encode(request.input.text).byteLength;
|
|
if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`);
|
|
return;
|
|
}
|
|
let size = 0;
|
|
const seen = new WeakSet<object>();
|
|
const pending: Array<{ value: unknown; depth: number }> = [{ value: request.input.value, depth: 0 }];
|
|
while (pending.length > 0) {
|
|
const { value, depth } = pending.pop()!;
|
|
if (depth > 128) throw new Error("Object input nesting exceeds maximum depth 128");
|
|
if (value === null || typeof value === "boolean") size += 4;
|
|
else if (typeof value === "number") {
|
|
if (!Number.isFinite(value)) throw new Error("Object input must contain finite JSON values");
|
|
size += 8;
|
|
} else if (typeof value === "string") size += new TextEncoder().encode(value).byteLength;
|
|
else {
|
|
if (typeof value !== "object") throw new Error("Object input must contain plain JSON values");
|
|
if (seen.has(value)) throw new Error("Object input must contain acyclic plain JSON values");
|
|
seen.add(value);
|
|
if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new Error("Object input must contain plain JSON objects");
|
|
const descriptors = Object.getOwnPropertyDescriptors(value);
|
|
for (const [key, descriptor] of Object.entries(descriptors)) {
|
|
if (key === "length") continue;
|
|
if (!("value" in descriptor)) throw new Error("Object input must contain plain JSON data properties");
|
|
size += new TextEncoder().encode(key).byteLength;
|
|
if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`);
|
|
pending.push({ value: descriptor.value, depth: depth + 1 });
|
|
}
|
|
}
|
|
if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`);
|
|
}
|
|
}
|
|
|
|
async function importInline(request: SourceImportRequest): Promise<SourceImportResult> {
|
|
validateBoundedInput(request);
|
|
if (request.sourceKind === "ci" && request.input.type === "object") {
|
|
const jobs = record(record(request.input.value)?.jobs);
|
|
if (!jobs || Object.keys(jobs).length === 0) throw new Error("Malformed CI input: jobs must be a non-empty object");
|
|
const names = Object.keys(jobs).sort(codePointCompare);
|
|
const nodes: DiagramNode[] = names.map((name) => ({ id: id("command", name), label: name, kind: "command", provenance: { path: request.path } }));
|
|
const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = [];
|
|
for (const name of names) {
|
|
const rawNeeds = record(jobs[name])?.needs;
|
|
const needs = (Array.isArray(rawNeeds) ? rawNeeds : rawNeeds === undefined ? [] : [rawNeeds]).filter((value): value is string => typeof value === "string").sort(codePointCompare);
|
|
for (const dependency of needs) {
|
|
if (!names.includes(dependency)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown CI job: ${dependency}`, path: request.path }); continue; }
|
|
edges.push({ id: id("edge", dependency, "needs", name, String(edges.length)), source: id("command", dependency), target: id("command", name), kind: "needs", provenance: { path: request.path } });
|
|
}
|
|
}
|
|
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|
|
if (request.sourceKind === "openapi" && request.input.type === "object") {
|
|
const root = record(request.input.value); const paths = record(root?.paths); const schemas = record(record(root?.components)?.schemas) ?? {};
|
|
if (!root || !paths) throw new Error("Malformed OpenAPI input: paths must be an object");
|
|
const nodes: DiagramNode[] = []; const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = [];
|
|
const methods = new Set(["get", "put", "post", "delete", "patch", "options", "head", "trace"]);
|
|
for (const path of Object.keys(paths).sort(codePointCompare)) {
|
|
const pathItem = record(paths[path]) ?? {};
|
|
for (const method of Object.keys(pathItem).filter((key) => methods.has(key.toLowerCase())).sort(codePointCompare)) {
|
|
const operationId = id("command", method, path);
|
|
nodes.push({ id: operationId, label: `${method.toUpperCase()} ${path}`, kind: "command", provenance: { path: request.path } });
|
|
const refs = new Set<string>(); collectSchemaRefs(pathItem[method], refs);
|
|
for (const schema of [...refs].sort(codePointCompare)) {
|
|
if (!Object.prototype.hasOwnProperty.call(schemas, schema)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown schema: ${schema}`, path: request.path }); continue; }
|
|
edges.push({ id: id("edge", method, path, schema, String(edges.length)), source: operationId, target: id("schema", schema), kind: "schema-reference", provenance: { path: request.path } });
|
|
}
|
|
}
|
|
}
|
|
for (const schema of Object.keys(schemas).sort(codePointCompare)) nodes.push({ id: id("schema", schema), label: schema, kind: "library", provenance: { path: request.path } });
|
|
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|
|
if (request.sourceKind === "sql" && request.input.type === "text") {
|
|
const scanned = maskComments(request.input.text, ["--"], false, "'");
|
|
const tables = new Map<string, number>(); const refs: Array<{ source: string; target: string; line: number }> = [];
|
|
const diagnostics: ImportDiagnostic[] = scanned.unknownLines.map((line) => ({ code: "unknown-construct", severity: "warning", message: "SQL construct is outside the bounded static subset", path: request.path, line }));
|
|
let current: string | undefined;
|
|
scanned.text.split(/\r?\n/).forEach((line, index) => {
|
|
const declaration = line.match(/^\s*CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?["`\[]?([A-Za-z_]\w*)/i)?.[1];
|
|
if (declaration) {
|
|
if (tables.has(declaration)) diagnostics.push({ code: "duplicate-id", severity: "warning", message: `Duplicate table: ${declaration}`, path: request.path, line: index + 1 });
|
|
else tables.set(declaration, index + 1);
|
|
current = declaration;
|
|
}
|
|
const target = line.match(/\bREFERENCES\s+["`\[]?([A-Za-z_]\w*)/i)?.[1];
|
|
if (current && target) refs.push({ source: current, target, line: index + 1 });
|
|
if (/\)\s*;/.test(line)) current = undefined;
|
|
});
|
|
const nodes: DiagramNode[] = [...tables].sort(([a], [b]) => codePointCompare(a, b)).map(([name, line]) => ({ id: id("table", name), label: name, kind: "table", provenance: { path: request.path, line } }));
|
|
const edges: DiagramEdge[] = refs.filter((ref) => tables.has(ref.target)).map((ref, occurrence) => ({ id: id("edge", ref.source, "fk", ref.target, String(occurrence)), source: id("table", ref.source), target: id("table", ref.target), kind: "foreign-key", provenance: { path: request.path, line: ref.line } }));
|
|
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|
|
if (request.sourceKind === "docker-compose" && request.input.type === "object") {
|
|
const root = record(request.input.value);
|
|
const services = record(root?.services);
|
|
const volumes = record(root?.volumes) ?? {};
|
|
if (!services || Object.keys(services).length === 0) throw new Error("Malformed Docker Compose input: services must be a non-empty object");
|
|
const serviceNames = Object.keys(services).sort(codePointCompare); const volumeNames = Object.keys(volumes).sort(codePointCompare);
|
|
const nodes: DiagramNode[] = [
|
|
...serviceNames.map((name) => ({ id: id("service", name), label: name, kind: "service", provenance: { path: request.path } })),
|
|
...volumeNames.map((name) => ({ id: id("volume", name), label: name, kind: "volume", provenance: { path: request.path } })),
|
|
];
|
|
const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = [];
|
|
for (const name of serviceNames) {
|
|
const service = record(services[name]) ?? {};
|
|
const depends = Array.isArray(service.depends_on) ? service.depends_on : Object.keys(record(service.depends_on) ?? {});
|
|
for (const dependency of depends.filter((value): value is string => typeof value === "string").sort(codePointCompare)) {
|
|
if (!serviceNames.includes(dependency)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown service: ${dependency}`, path: request.path }); continue; }
|
|
edges.push({ id: id("edge", name, "depends", dependency, String(edges.length)), source: id("service", name), target: id("service", dependency), kind: "depends-on", provenance: { path: request.path } });
|
|
}
|
|
for (const mount of (Array.isArray(service.volumes) ? service.volumes : []).filter((value): value is string => typeof value === "string").sort(codePointCompare)) {
|
|
const volume = mount.split(":", 1)[0];
|
|
if (volumeNames.includes(volume)) edges.push({ id: id("edge", name, "mounts", volume, String(edges.length)), source: id("service", name), target: id("volume", volume), kind: "mounts", provenance: { path: request.path } });
|
|
}
|
|
}
|
|
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|
|
if (request.sourceKind === "kubernetes" && request.input.type === "object") {
|
|
const values = Array.isArray(request.input.value) ? request.input.value : [request.input.value];
|
|
const rawObjects = values.map(record).filter((value): value is Record<string, unknown> => Boolean(value && typeof value.kind === "string" && objectName(value)));
|
|
const occurrences = new Map<string, number>();
|
|
const objects = rawObjects.map((value) => {
|
|
const kind = String(value.kind);
|
|
const name = objectName(value)!;
|
|
const rawNamespace = record(value.metadata)?.namespace;
|
|
const namespace = typeof rawNamespace === "string" && rawNamespace.length > 0 ? rawNamespace : "default";
|
|
const key = JSON.stringify([kind, namespace, name]);
|
|
const occurrence = occurrences.get(key) ?? 0;
|
|
occurrences.set(key, occurrence + 1);
|
|
return { value, kind, name, namespace, nodeId: id("k8s", kind, namespace, name, String(occurrence)) };
|
|
});
|
|
const diagnostics: ImportDiagnostic[] = objects.filter((object) => object.kind === "Secret").map(() => ({ code: "secret-redacted", severity: "warning", message: "Kubernetes Secret payload omitted", path: request.path }));
|
|
const nodes: DiagramNode[] = objects.map((object) => ({
|
|
id: object.nodeId,
|
|
label: `${object.kind}/${object.namespace}/${object.name}`,
|
|
kind: object.kind.toLowerCase(),
|
|
provenance: { path: request.path },
|
|
})).sort((a, b) => codePointCompare(a.label, b.label));
|
|
const byKey = new Map<string, typeof objects[number]>();
|
|
for (const object of objects) {
|
|
const key = JSON.stringify([object.kind, object.namespace, object.name]);
|
|
if (!byKey.has(key)) byKey.set(key, object);
|
|
}
|
|
const workloadKinds = new Set(["Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "Job"]);
|
|
const edges: DiagramEdge[] = [];
|
|
for (const object of objects) {
|
|
const secretRefs = new Set<string>();
|
|
if (workloadKinds.has(object.kind)) collectWorkloadSecretRefs(record(record(object.value.spec)?.template)?.spec, secretRefs);
|
|
for (const target of [...secretRefs].sort(codePointCompare)) {
|
|
const secret = byKey.get(JSON.stringify(["Secret", object.namespace, target]));
|
|
if (secret) edges.push({ id: id("edge", object.nodeId, "secret", secret.nodeId, String(edges.length)), source: object.nodeId, target: secret.nodeId, kind: "uses-secret", provenance: { path: request.path } });
|
|
}
|
|
if (object.kind === "Service") {
|
|
const selector = record(record(object.value.spec)?.selector);
|
|
if (!selector || Object.keys(selector).length === 0) continue;
|
|
for (const workload of objects.filter((candidate) => workloadKinds.has(candidate.kind) && candidate.namespace === object.namespace)) {
|
|
const labels = record(record(record(record(workload.value.spec)?.template)?.metadata)?.labels);
|
|
if (labels && Object.entries(selector).every(([key, selected]) => labels[key] === selected)) {
|
|
edges.push({ id: id("edge", object.nodeId, "selects", workload.nodeId, String(edges.length)), source: object.nodeId, target: workload.nodeId, kind: "selects", provenance: { path: request.path } });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|
|
if (request.sourceKind === "terraform" && request.input.type === "text") {
|
|
const heredocs = maskTerraformHeredocs(request.input.text);
|
|
const structural = maskComments(heredocs.text, ["#", "//"]);
|
|
const referencesOnly = maskTerraformStrings(structural.text);
|
|
const unknownLines = [...new Set([...heredocs.unknownLines, ...structural.unknownLines, ...referencesOnly.unknownLines])].sort((a, b) => a - b);
|
|
const diagnostics: ImportDiagnostic[] = unknownLines.map((line) => ({ code: "unknown-construct", severity: "warning", message: "Terraform construct is outside the bounded static subset", path: request.path, line }));
|
|
const resources = new Map<string, number>();
|
|
const references: Array<{ source: string; target: string; line: number }> = [];
|
|
const referenceLines = referencesOnly.text.split(/\r?\n/);
|
|
let current: string | undefined;
|
|
structural.text.split(/\r?\n/).forEach((line, index) => {
|
|
const declaration = line.match(/^\s*resource\s+"([^"]+)"\s+"([^"]+)"\s*\{/);
|
|
if (declaration) {
|
|
current = `${declaration[1]}.${declaration[2]}`;
|
|
resources.set(current, index + 1);
|
|
if (/\{\s*\}/.test(line)) current = undefined;
|
|
return;
|
|
}
|
|
if (current) {
|
|
for (const match of referenceLines[index].matchAll(/\b([A-Za-z_]\w*\.[A-Za-z_]\w*)\.[A-Za-z_]\w*/g)) references.push({ source: current, target: match[1], line: index + 1 });
|
|
if (/^\s*}/.test(line)) current = undefined;
|
|
}
|
|
});
|
|
const nodes = [...resources].sort(([a], [b]) => codePointCompare(a, b)).map(([name, line]) => ({ id: id("resource", name), label: name, kind: "resource", provenance: { path: request.path, line } }));
|
|
const edges = references.filter((ref) => resources.has(ref.target)).sort((a, b) => codePointCompare(a.source, b.source) || codePointCompare(a.target, b.target) || a.line - b.line).map((ref, occurrence) => ({ id: id("edge", ref.source, ref.target, String(occurrence)), source: id("resource", ref.source), target: id("resource", ref.target), kind: "reference", provenance: { path: request.path, line: ref.line } }));
|
|
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|
|
if (!(["python", "javascript", "typescript", "go", "rust"] as SourceKind[]).includes(request.sourceKind) || request.input.type !== "text") throw new Error("Unsupported source input");
|
|
const nodes: DiagramNode[] = [{ id: id("module", request.path), label: request.path, kind: "module", provenance: { path: request.path, line: 1 } }];
|
|
const edges: DiagramEdge[] = [];
|
|
const diagnostics: ImportDiagnostic[] = [];
|
|
const javaScriptScan = request.sourceKind === "javascript" || request.sourceKind === "typescript" ? scanJavaScriptImports(request.input.text) : undefined;
|
|
const nonJavaScriptScan = request.sourceKind === "python" ? maskPythonTripleStrings(request.input.text)
|
|
: request.sourceKind === "go" ? maskComments(request.input.text, ["//"], false, "`")
|
|
: request.sourceKind === "rust" ? maskComments(request.input.text, ["//"], true, "", true)
|
|
: { text: request.input.text, unknownLines: [] };
|
|
for (const line of javaScriptScan?.unknownLines ?? nonJavaScriptScan.unknownLines) diagnostics.push({ code: "unknown-construct", severity: "warning", message: `${request.sourceKind} construct is outside the bounded static-import subset`, path: request.path, line });
|
|
nonJavaScriptScan.text.split(/\r?\n/).forEach((line, index) => {
|
|
const dynamicPython = request.sourceKind === "python" && /\b(?:__import__|importlib\.import_module)\s*\(\s*[^"']/.test(line);
|
|
if (dynamicPython) diagnostics.push({ code: "unknown-construct", severity: "warning", message: "Dynamic imports are not evaluated", path: request.path, line: index + 1 });
|
|
});
|
|
const seen = new Set<string>();
|
|
const foundImports = request.sourceKind === "rust"
|
|
? nonJavaScriptScan.text.split(/\r?\n/).flatMap((line, index) => {
|
|
const mod = line.match(/^\s*mod\s+([A-Za-z_]\w*)\s*;/)?.[1];
|
|
const use = line.match(/^\s*use\s+([^;]+);/)?.[1];
|
|
if (/\w+!\s*\(/.test(line)) diagnostics.push({ code: "unknown-construct", severity: "warning", message: "Rust macros are not expanded", path: request.path, line: index + 1 });
|
|
if (mod) return [{ name: mod, line: index + 1 }];
|
|
if (use) {
|
|
const root = use.includes("::{") ? use.slice(0, use.indexOf("::{")) : use.split("::").slice(0, -1).join("::") || use;
|
|
return [{ name: root, line: index + 1 }];
|
|
}
|
|
return [];
|
|
})
|
|
: javaScriptScan?.imports ?? codeImports(request.sourceKind as "python" | "go", nonJavaScriptScan.text);
|
|
for (const found of foundImports) {
|
|
if (seen.has(found.name)) continue;
|
|
seen.add(found.name);
|
|
const targetId = id("library", found.name);
|
|
nodes.push({ id: targetId, label: found.name, kind: "library", provenance: { path: request.path, line: found.line } });
|
|
edges.push({ id: id("edge", request.path, found.name, String(edges.length)), source: nodes[0].id, target: targetId, kind: "import", provenance: { path: request.path, line: found.line } });
|
|
}
|
|
return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|
|
|
|
function isConfined(root: string, candidate: string): boolean {
|
|
const pathFromRoot = relative(root, candidate);
|
|
return pathFromRoot === "" || (!pathFromRoot.startsWith("..") && !isAbsolute(pathFromRoot));
|
|
}
|
|
|
|
async function openConfined(candidate: string, root: string): Promise<{ handle: FileHandle; canonical: string }> {
|
|
let handle: FileHandle;
|
|
try {
|
|
handle = await open(candidate, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === "ELOOP") throw new Error("Source symlink escape risk cannot be opened safely");
|
|
throw error;
|
|
}
|
|
try {
|
|
const canonical = await realpath(`/proc/self/fd/${handle.fd}`);
|
|
if (!isConfined(root, canonical)) throw new Error("Opened source escapes outside declared root");
|
|
return { handle, canonical };
|
|
} catch (error) {
|
|
await handle.close();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function readBounded(handle: FileHandle, limit: number, directory: boolean): Promise<Buffer> {
|
|
const initial = await handle.stat();
|
|
if (!initial.isFile()) throw new Error("Source entry must be a regular file");
|
|
if (initial.size > limit) throw new Error(`${directory ? "Directory source input" : "Source input"} exceeds maxBytes ${limit}`);
|
|
const chunks: Buffer[] = [];
|
|
let consumed = 0;
|
|
while (consumed < limit) {
|
|
const chunk = Buffer.allocUnsafe(Math.min(65_536, limit - consumed));
|
|
const { bytesRead } = await handle.read(chunk, 0, chunk.byteLength, null);
|
|
if (bytesRead === 0) break;
|
|
chunks.push(chunk.subarray(0, bytesRead));
|
|
consumed += bytesRead;
|
|
}
|
|
const final = await handle.stat();
|
|
if (initial.dev !== final.dev || initial.ino !== final.ino) throw new Error("Opened source identity changed during read");
|
|
if (final.size > limit || consumed < initial.size) throw new Error(`${directory ? "Directory source input" : "Source input"} exceeds maxBytes ${limit}`);
|
|
return Buffer.concat(chunks, consumed);
|
|
}
|
|
|
|
export async function importSource(request: SourceImportRequest): Promise<SourceImportResult> {
|
|
validateBoundedInput(request);
|
|
if (request.input.type !== "file") return importInline(request);
|
|
if (isAbsolute(request.path)) throw new Error("Source path escapes outside declared root");
|
|
const rootPath = await realpath(request.input.root);
|
|
const lexicalPath = resolve(rootPath, request.path);
|
|
if (!isConfined(rootPath, lexicalPath)) throw new Error("Source path escapes outside declared root");
|
|
const limit = request.maxBytes ?? 1_048_576;
|
|
const objectKinds = new Set<SourceKind>(["kubernetes", "docker-compose", "openapi", "ci"]);
|
|
const extensions: Record<SourceKind, Set<string>> = {
|
|
python: new Set([".py"]), javascript: new Set([".js", ".jsx", ".mjs", ".cjs"]), typescript: new Set([".ts", ".tsx", ".mts", ".cts"]),
|
|
go: new Set([".go"]), rust: new Set([".rs"]), terraform: new Set([".tf"]), sql: new Set([".sql"]),
|
|
kubernetes: new Set([".json", ".yaml", ".yml"]), "docker-compose": new Set([".json", ".yaml", ".yml"]), openapi: new Set([".json", ".yaml", ".yml"]), ci: new Set([".json", ".yaml", ".yml"]),
|
|
};
|
|
const parse = async (buffer: Buffer, provenancePath: string): Promise<SourceImportResult> => {
|
|
const text = new TextDecoder("utf-8", { fatal: true }).decode(buffer);
|
|
let input: SourceInput;
|
|
if (objectKinds.has(request.sourceKind)) {
|
|
const yaml = /\.ya?ml$/i.test(provenancePath);
|
|
try {
|
|
if (yaml) {
|
|
const documents = loadAll(text, undefined, { schema: JSON_SCHEMA, json: false }).filter((value) => value !== undefined);
|
|
if (documents.length === 0) throw new Error("empty");
|
|
if (request.sourceKind !== "kubernetes" && documents.length !== 1) throw new Error("multiple documents");
|
|
input = { type: "object", value: request.sourceKind === "kubernetes" ? documents : documents[0] };
|
|
} else input = { type: "object", value: JSON.parse(text) as unknown };
|
|
} catch { throw new Error(`Malformed ${yaml ? "YAML or disallowed tag" : "JSON"} source: ${provenancePath}`); }
|
|
} else input = { type: "text", text };
|
|
return importInline({ ...request, path: provenancePath, input });
|
|
};
|
|
const source = await openConfined(lexicalPath, rootPath);
|
|
const info = await source.handle.stat();
|
|
if (info.isFile()) {
|
|
try { return await parse(await readBounded(source.handle, limit, false), request.path); }
|
|
finally { await source.handle.close(); }
|
|
}
|
|
if (!info.isDirectory()) { await source.handle.close(); throw new Error("Source path must be a file or directory"); }
|
|
|
|
const results: SourceImportResult[] = [];
|
|
let consumed = 0;
|
|
const walk = async (directory: FileHandle): Promise<void> => {
|
|
for (const name of (await readdir(`/proc/self/fd/${directory.fd}`)).sort(codePointCompare)) {
|
|
const opened = await openConfined(resolve(`/proc/self/fd/${directory.fd}`, name), rootPath);
|
|
try {
|
|
const entryInfo = await opened.handle.stat();
|
|
if (entryInfo.isDirectory()) await walk(opened.handle);
|
|
else if (entryInfo.isFile()) {
|
|
const dot = name.lastIndexOf("."); const extension = dot >= 0 ? name.slice(dot).toLowerCase() : "";
|
|
if (extensions[request.sourceKind].has(extension)) {
|
|
const provenancePath = relative(rootPath, opened.canonical).split("\\").join("/");
|
|
const buffer = await readBounded(opened.handle, limit - consumed, true);
|
|
consumed += buffer.byteLength;
|
|
results.push(await parse(buffer, provenancePath));
|
|
}
|
|
} else throw new Error("Source directory contains an unsupported entry type");
|
|
} finally { await opened.handle.close(); }
|
|
}
|
|
};
|
|
try { await walk(source.handle); }
|
|
finally { await source.handle.close(); }
|
|
if (results.length === 0) throw new Error(`No supported ${request.sourceKind} source files found`);
|
|
const nodeById = new Map<string, DiagramNode>(); const edgeById = new Map<string, DiagramEdge>(); const diagnostics: ImportDiagnostic[] = [];
|
|
for (const result of results) {
|
|
diagnostics.push(...result.diagnostics);
|
|
for (const node of result.diagram.pages[0].nodes) if (!nodeById.has(node.id)) nodeById.set(node.id, node);
|
|
for (const edge of result.diagram.pages[0].edges) {
|
|
if (edgeById.has(edge.id)) diagnostics.push({ code: "duplicate-id", severity: "warning", message: `Duplicate edge omitted: ${edge.id}`, path: String(edge.provenance?.path ?? request.path) });
|
|
else edgeById.set(edge.id, edge);
|
|
}
|
|
}
|
|
return { diagram: diagramFor(request, [...nodeById.values()], [...edgeById.values()]), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } };
|
|
}
|