diff --git a/scripts/src/services/source-importers/index.test.ts b/scripts/src/services/source-importers/index.test.ts new file mode 100644 index 0000000..3100215 --- /dev/null +++ b/scripts/src/services/source-importers/index.test.ts @@ -0,0 +1,385 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { importSource } from "./index.js"; +import { validateDiagramIR } from "../../model/diagram-ir.js"; + +const textInput = (sourceKind: Parameters[0]["sourceKind"], path: string, text: string) => + importSource({ sourceKind, path, input: { type: "text", text } }); + +test("imports Python modules with exact line provenance", async () => { + const result = await textInput("python", "src/app.py", "import os\nfrom pkg.db import Client\n"); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => [n.label, n.kind]), [ + ["src/app.py", "module"], ["os", "library"], ["pkg.db", "library"], + ]); + assert.equal(result.diagram.pages[0].edges[1].provenance?.line, 2); + assert.equal(validateDiagramIR(result.diagram), result.diagram); +}); + +test("imports JavaScript and TypeScript static and require dependencies", async () => { + const js = await textInput("javascript", "src/app.js", "import express from 'express';\nconst x = require('./local.js');\n"); + const ts = await textInput("typescript", "src/app.ts", "export { x } from '@scope/pkg';\nimport type { T } from './types.js';\n"); + assert.deepEqual(js.diagram.pages[0].nodes.slice(1).map((n) => n.label), ["express", "./local.js"]); + assert.deepEqual(ts.diagram.pages[0].nodes.slice(1).map((n) => n.label), ["@scope/pkg", "./types.js"]); + assert.ok(js.diagram.pages[0].nodes.every((n) => ["module", "library"].includes(n.kind ?? ""))); +}); + +test("JavaScript scanner ignores comments and strings while preserving real static imports", async () => { + const source = [ + "// import commented from 'commented';", + "/* require('blocked');", + " export { x } from 'also-blocked'; */", + "const text = \"import fake from 'inside-string'\";", + "const template = `require('inside-template')`;", + "const pattern = /import regexFake from 'inside-regex'/;", + "loader.require('member-call');", + "const metadata = { import: 'property', from: 'property-from' };", + "import real from 'real';", + "export { thing } from './actual.js';", + "const required = require(\"required\");", + "import type {", + " Value", + "} from './types.js';", + "// const dynamic = import(name);", + ].join("\n"); + const result = await textInput("typescript", "src/app.ts", source); + assert.deepEqual(result.diagram.pages[0].nodes.slice(1).map((node) => node.label), ["real", "./actual.js", "required", "./types.js"]); + assert.deepEqual(result.diagnostics, []); +}); + +test("JavaScript scanner treats arrow-expression regex literals as inert without hiding division or imports", async () => { + const source = [ + "const matcher = () => /require('phantom')/;", + "const ratio = numerator / denominator;", + "import real from 'real';", + "const actual = require('./actual.js');", + ].join("\n"); + const result = await textInput("typescript", "src/regex.ts", source); + assert.deepEqual(result.diagram.pages[0].nodes.slice(1).map((node) => node.label), ["real", "./actual.js"]); + assert.deepEqual(result.diagnostics, []); +}); + +test("JavaScript scanner diagnoses unsupported dynamic and unterminated constructs", async () => { + const result = await textInput("javascript", "src/app.js", "const dynamic = import(name);\nconst template = `${load(name)}`;\n/* unterminated"); + assert.ok(result.diagnostics.length >= 3); + assert.ok(result.diagnostics.every((diagnostic) => diagnostic.code === "unknown-construct")); +}); + +test("imports Go single and grouped imports", async () => { + const result = await textInput("go", "cmd/main.go", "package main\nimport \"fmt\"\nimport (\n alias \"example.com/acme/lib\"\n _ \"net/http/pprof\"\n)\n"); + assert.deepEqual(result.diagram.pages[0].nodes.slice(1).map((n) => n.label), ["fmt", "example.com/acme/lib", "net/http/pprof"]); + assert.equal(result.diagram.pages[0].edges[2].provenance?.line, 5); +}); + +test("imports Rust mod declarations and use roots without guessing macros", async () => { + const result = await textInput("rust", "src/lib.rs", "mod api;\nuse crate::db::Pool;\nuse serde::{Serialize, Deserialize};\ncustom!(unknown);\n"); + assert.deepEqual(result.diagram.pages[0].nodes.slice(1).map((n) => n.label), ["api", "crate::db", "serde"]); + assert.equal(result.diagnostics[0].code, "unknown-construct"); +}); + +test("imports Terraform resources and explicit references", async () => { + const result = await textInput("terraform", "main.tf", "resource \"aws_vpc\" \"main\" {}\nresource \"aws_subnet\" \"web\" {\n vpc_id = aws_vpc.main.id\n}\n"); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => n.label), ["aws_subnet.web", "aws_vpc.main"]); + assert.deepEqual(result.diagram.pages[0].edges.map((e) => [e.source, e.target, e.kind]), [["resource-6177735f7375626e65742e776562", "resource-6177735f7670632e6d61696e", "reference"]]); + assert.equal(result.diagram.pages[0].edges[0].provenance?.line, 3); +}); + +test("non-JavaScript scanners ignore inert constructs and diagnose unterminated regions", async () => { + const [python, unterminatedPython, go, rust, sql, terraform] = await Promise.all([ + textInput("python", "app.py", "'''\nimport phantom\n'''\nimport real\n"), + textInput("python", "broken.py", "\"\"\"unterminated\nimport phantom\n"), + textInput("go", "main.go", "package main\n/*\nimport \"phantom\"\n*/\nimport \"fmt\"\n/* unterminated\n"), + textInput("rust", "lib.rs", "/*\nmod phantom;\nuse fake::Thing;\n*/\nmod api;\nuse serde::Serialize;\n/* unterminated\n"), + textInput("sql", "schema.sql", "/*\nCREATE TABLE phantom (id INT REFERENCES ghost(id));\n*/\nCREATE TABLE real (id INT);\n/* unterminated\n"), + textInput("terraform", "main.tf", [ + "/*", "resource \"aws_vpc\" \"phantom\" {}", "*/", + "resource \"aws_vpc\" \"main\" {}", + "resource \"aws_subnet\" \"web\" {", + " description = \"aws_vpc.phantom.id\"", + " # fake = aws_vpc.phantom.id", + " vpc_id = aws_vpc.main.id", + "}", "/* unterminated", + ].join("\n")), + ]); + assert.deepEqual(python.diagram.pages[0].nodes.map((node) => node.label), ["app.py", "real"]); + assert.deepEqual(unterminatedPython.diagram.pages[0].nodes.map((node) => node.label), ["broken.py"]); + assert.deepEqual(go.diagram.pages[0].nodes.map((node) => node.label), ["main.go", "fmt"]); + assert.deepEqual(rust.diagram.pages[0].nodes.map((node) => node.label), ["lib.rs", "api", "serde"]); + assert.deepEqual(sql.diagram.pages[0].nodes.map((node) => node.label), ["real"]); + assert.deepEqual(terraform.diagram.pages[0].nodes.map((node) => node.label), ["aws_subnet.web", "aws_vpc.main"]); + assert.equal(terraform.diagram.pages[0].edges.length, 1); + for (const result of [unterminatedPython, go, rust, sql, terraform]) { + assert.ok(result.diagnostics.some((diagnostic) => diagnostic.code === "unknown-construct")); + } +}); + +test("multiline literals mask phantom topology and diagnose only unterminated regions", async () => { + const [go, rust, sql, terraform] = await Promise.all([ + textInput("go", "main.go", [ + "package main", "var first = `", "import \"phantom\"", "`", "import \"fmt\"", + ].join("\n")), + textInput("rust", "lib.rs", [ + "const FIRST: &str = r\"", "mod phantom_one;", "\";", + "const SECOND: &str = r##\"", "use phantom_two::Thing;", "\"##;", + "mod api;", "use serde::Serialize;", + ].join("\n")), + textInput("sql", "schema.sql", [ + "CREATE TABLE real (id INT);", "INSERT INTO real(note) VALUES ('", + "CREATE TABLE phantom (id INT REFERENCES ghost(id));", "escaped '' quote", "');", + ].join("\n")), + textInput("terraform", "main.tf", [ + "resource \"aws_vpc\" \"main\" {}", "resource \"aws_subnet\" \"web\" {", + " user_data = <<-EOT", " resource \"aws_instance\" \"phantom\" {}", + " aws_vpc.phantom.id", " EOT", " vpc_id = aws_vpc.main.id", "}", + ].join("\n")), + ]); + assert.deepEqual(go.diagram.pages[0].nodes.map((node) => node.label), ["main.go", "fmt"]); + assert.deepEqual(rust.diagram.pages[0].nodes.map((node) => node.label), ["lib.rs", "api", "serde"]); + assert.deepEqual(sql.diagram.pages[0].nodes.map((node) => node.label), ["real"]); + assert.deepEqual(terraform.diagram.pages[0].nodes.map((node) => node.label), ["aws_subnet.web", "aws_vpc.main"]); + assert.equal(terraform.diagram.pages[0].edges.length, 1); + for (const result of [go, rust, sql, terraform]) assert.deepEqual(result.diagnostics, []); + + const unterminated = await Promise.all([ + textInput("go", "broken.go", "package main\nvar value = `unterminated\nimport \"phantom\""), + textInput("rust", "broken.rs", "const VALUE: &str = r###\"unterminated\nmod phantom;"), + textInput("sql", "broken.sql", "CREATE TABLE real (id INT);\nINSERT INTO real(note) VALUES ('unterminated\nCREATE TABLE phantom (id INT);"), + textInput("terraform", "broken.tf", "resource \"aws_vpc\" \"main\" {}\nvalue = < node.label.includes("phantom")), false); + assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code), ["unknown-construct"]); + } +}); + +test("imports Kubernetes relationships and redacts Secret payloads", async () => { + const objects = [ + { apiVersion: "v1", kind: "Secret", metadata: { name: "db-secret" }, data: { password: "c2VjcmV0" }, stringData: { token: "plain" } }, + { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: "api" }, spec: { selector: { matchLabels: { app: "api" } }, template: { metadata: { labels: { app: "api" } }, spec: { containers: [{ envFrom: [{ secretRef: { name: "db-secret" } }] }] } } } }, + { apiVersion: "v1", kind: "Service", metadata: { name: "api-svc" }, spec: { selector: { app: "api" } } }, + ]; + const result = await importSource({ sourceKind: "kubernetes", path: "k8s.json", input: { type: "object", value: objects } }); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => n.label), ["Deployment/default/api", "Secret/default/db-secret", "Service/default/api-svc"]); + assert.deepEqual(result.diagram.pages[0].edges.map((e) => e.kind), ["uses-secret", "selects"]); + assert.equal(JSON.stringify(result).includes("c2VjcmV0"), false); + assert.equal(JSON.stringify(result).includes("plain"), false); + assert.equal(result.diagnostics[0].code, "secret-redacted"); +}); + +test("Kubernetes ignores secretRef-shaped data outside supported workload fields", async () => { + const objects = [ + { kind: "Secret", metadata: { name: "real" } }, + { kind: "Secret", metadata: { name: "phantom" } }, + { + kind: "Deployment", metadata: { name: "api" }, status: { secretRef: { name: "phantom" } }, + spec: { template: { spec: { containers: [{ envFrom: [{ secretRef: { name: "real" } }] }] } } }, + }, + ]; + const result = await importSource({ sourceKind: "kubernetes", path: "k8s.json", input: { type: "object", value: objects } }); + const labelById = new Map(result.diagram.pages[0].nodes.map((node) => [node.id, node.label])); + assert.deepEqual(result.diagram.pages[0].edges.map((edge) => [edge.kind, labelById.get(edge.target)]), [ + ["uses-secret", "Secret/default/real"], + ]); +}); + +test("scopes Kubernetes identity, secret references, and non-empty Service selectors by namespace", async () => { + const objects = [ + { kind: "Secret", metadata: { name: "shared" } }, + { kind: "Secret", metadata: { namespace: "team-b", name: "shared" } }, + { kind: "Deployment", metadata: { name: "api" }, spec: { selector: { matchLabels: { selectorOnly: "wrong" } }, template: { metadata: { labels: { app: "api" } }, spec: { containers: [{ envFrom: [{ secretRef: { name: "shared" } }] }] } } } }, + { kind: "Deployment", metadata: { namespace: "team-b", name: "api" }, spec: { template: { metadata: { labels: { app: "api" } }, spec: { containers: [{ envFrom: [{ secretRef: { name: "shared" } }] }] } } } }, + { kind: "Deployment", metadata: { name: "selector-only" }, spec: { selector: { matchLabels: { app: "api" } }, template: { metadata: { labels: { app: "other" } } } } }, + { kind: "Service", metadata: { name: "api" }, spec: { selector: { app: "api" } } }, + { kind: "Service", metadata: { name: "empty" }, spec: { selector: {} } }, + ]; + const result = await importSource({ sourceKind: "kubernetes", path: "k8s.json", input: { type: "object", value: objects } }); + const nodes = result.diagram.pages[0].nodes; + assert.equal(new Set(nodes.map((node) => node.id)).size, nodes.length); + assert.ok(nodes.some((node) => node.label === "Secret/default/shared")); + assert.ok(nodes.some((node) => node.label === "Secret/team-b/shared")); + const labelById = new Map(nodes.map((node) => [node.id, node.label])); + const relationships = result.diagram.pages[0].edges.map((edge) => [labelById.get(edge.source), edge.kind, labelById.get(edge.target)]); + assert.deepEqual(relationships, [ + ["Deployment/default/api", "uses-secret", "Secret/default/shared"], + ["Deployment/team-b/api", "uses-secret", "Secret/team-b/shared"], + ["Service/default/api", "selects", "Deployment/default/api"], + ]); +}); + +test("imports Docker Compose service dependencies and named volumes", async () => { + const result = await importSource({ sourceKind: "docker-compose", path: "compose.json", input: { type: "object", value: { + services: { api: { depends_on: ["db"], volumes: ["data:/var/lib/app"] }, db: { image: "postgres" } }, volumes: { data: {} }, + } } }); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => [n.label, n.kind]), [["api", "service"], ["db", "service"], ["data", "volume"]]); + assert.deepEqual(result.diagram.pages[0].edges.map((e) => e.kind), ["depends-on", "mounts"]); +}); + +test("imports SQL tables and foreign keys", async () => { + const result = await textInput("sql", "schema.sql", "CREATE TABLE users (id INT PRIMARY KEY);\nCREATE TABLE orders (\n user_id INT,\n FOREIGN KEY (user_id) REFERENCES users(id)\n);\n"); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => n.label), ["orders", "users"]); + assert.deepEqual(result.diagram.pages[0].edges.map((e) => [e.source, e.target, e.kind]), [["table-6f7264657273", "table-7573657273", "foreign-key"]]); + assert.equal(result.diagram.pages[0].edges[0].provenance?.line, 4); +}); + +test("orders importer output by Unicode code point without locale-sensitive comparison", async () => { + const sql = await textInput("sql", "schema.sql", "CREATE TABLE a (id INT);\nCREATE TABLE Z (id INT);\n"); + assert.deepEqual(sql.diagram.pages[0].nodes.map((node) => node.label), ["Z", "a"]); + const kubernetes = await importSource({ sourceKind: "kubernetes", path: "k8s.json", input: { type: "object", value: [ + { kind: "Service", metadata: { name: "a" }, spec: { selector: {} } }, + { kind: "Service", metadata: { name: "Z" }, spec: { selector: {} } }, + { kind: "Service", metadata: { name: "😀" }, spec: { selector: {} } }, + { kind: "Service", metadata: { name: "\uE000" }, spec: { selector: {} } }, + ] } }); + assert.deepEqual(kubernetes.diagram.pages[0].nodes.map((node) => node.label), ["Service/default/Z", "Service/default/a", "Service/default/\uE000", "Service/default/😀"]); +}); + +test("imports OpenAPI operations and schema references", async () => { + const result = await importSource({ sourceKind: "openapi", path: "openapi.json", input: { type: "object", value: { + openapi: "3.1.0", paths: { "/pets": { get: { operationId: "listPets", responses: { "200": { content: { "application/json": { schema: { $ref: "#/components/schemas/Pet" } } } } } } } }, + components: { schemas: { Pet: { type: "object" } } }, + } } }); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => [n.label, n.kind]), [["GET /pets", "command"], ["Pet", "library"]]); + assert.equal(result.diagram.pages[0].edges[0].kind, "schema-reference"); +}); + +test("OpenAPI inherited schema names remain unknown references", async () => { + const result = await importSource({ sourceKind: "openapi", path: "openapi.json", input: { type: "object", value: { + paths: { "/unsafe": { get: { responses: { "200": { content: { "application/json": { schema: { $ref: "#/components/schemas/toString" } } } } } } } }, + components: { schemas: {} }, + } } }); + assert.deepEqual(result.diagram.pages[0].nodes.map((node) => node.label), ["GET /unsafe"]); + assert.deepEqual(result.diagram.pages[0].edges, []); + assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code), ["unknown-reference"]); +}); + +test("imports CI job DAG needs", async () => { + const result = await importSource({ sourceKind: "ci", path: ".github/workflows/ci.json", input: { type: "object", value: { + jobs: { deploy: { needs: ["build", "test"] }, test: { needs: "build" }, build: { "runs-on": "ubuntu-latest" } }, + } } }); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => [n.label, n.kind]), [["build", "command"], ["deploy", "command"], ["test", "command"]]); + assert.deepEqual(result.diagram.pages[0].edges.map((e) => [e.source, e.target]), [["command-6275696c64", "command-6465706c6f79"], ["command-74657374", "command-6465706c6f79"], ["command-6275696c64", "command-74657374"]]); +}); + +test("keeps generated IDs unique for normalized names, ambiguous composites, and repeated relationships", async () => { + const result = await importSource({ sourceKind: "ci", path: "ci.json", input: { type: "object", value: { + jobs: { + A: {}, + "A": {}, + a: {}, + "a-needs-b": {}, + "b-needs-c": { needs: ["a", "a"] }, + c: { needs: "a-needs-b" }, + }, + } } }); + const nodes = result.diagram.pages[0].nodes; + const edges = result.diagram.pages[0].edges; + assert.equal(new Set(nodes.map((node) => node.id)).size, nodes.length); + assert.equal(new Set(edges.map((edge) => edge.id)).size, edges.length); + assert.equal(edges.length, 3); + assert.doesNotThrow(() => validateDiagramIR(result.diagram)); +}); + +test("keeps sanitized IDs unique and reports duplicate logical definitions", async () => { + const openapi = await importSource({ sourceKind: "openapi", path: "api.json", input: { type: "object", value: { paths: { "/x": { get: {} } }, components: { schemas: { "A/B": {}, "A-B": {} } } } } }); + assert.equal(new Set(openapi.diagram.pages[0].nodes.map((n) => n.id)).size, openapi.diagram.pages[0].nodes.length); + const sql = await textInput("sql", "schema.sql", "CREATE TABLE users (id INT);\nCREATE TABLE users (id INT);\n"); + assert.equal(sql.diagnostics[0].code, "duplicate-id"); + assert.doesNotThrow(() => validateDiagramIR(openapi.diagram)); +}); + +test("disambiguates every repeated relationship occurrence deterministically", async () => { + const sqlText = "CREATE TABLE users (id INT PRIMARY KEY);\nCREATE TABLE orders (\n a INT REFERENCES users(id),\n b INT REFERENCES users(id)\n);\n"; + const terraformText = "resource \"aws_vpc\" \"main\" {}\nresource \"aws_subnet\" \"web\" {\n first = aws_vpc.main.id\n second = aws_vpc.main.id\n}\n"; + const composeValue = { services: { api: { depends_on: ["db", "db"], volumes: ["data:/one", "data:/two"] }, db: {} }, volumes: { data: {} } }; + const results = await Promise.all([ + textInput("sql", "schema.sql", sqlText), + textInput("terraform", "main.tf", terraformText), + importSource({ sourceKind: "docker-compose", path: "compose.json", input: { type: "object", value: composeValue } }), + ]); + const expectedEdgeCounts = [2, 2, 4]; + results.forEach((result, index) => { + const edges = result.diagram.pages[0].edges; + assert.equal(edges.length, expectedEdgeCounts[index]); + assert.equal(new Set(edges.map((edge) => edge.id)).size, edges.length); + assert.doesNotThrow(() => validateDiagramIR(result.diagram)); + }); + const again = await textInput("terraform", "main.tf", terraformText); + assert.deepEqual(again.diagram.pages[0].edges.map((edge) => edge.id), results[1].diagram.pages[0].edges.map((edge) => edge.id)); +}); + +test("rejects oversized and malformed inputs without executing object values", async () => { + await assert.rejects(() => importSource({ sourceKind: "python", path: "x.py", maxBytes: 4, input: { type: "text", text: "import os" } }), /exceeds.*4/i); + await assert.rejects(() => importSource({ sourceKind: "ci", path: "ci.json", input: { type: "object", value: { jobs: null } } }), /malformed CI/i); + let accessed = false; + const tagged = Object.create({ yamlTag: "!exec" }) as Record; + Object.defineProperty(tagged, "jobs", { enumerable: true, get() { accessed = true; return {}; } }); + await assert.rejects(() => importSource({ sourceKind: "ci", path: "ci.json", input: { type: "object", value: tagged } }), /plain JSON/i); + assert.equal(accessed, false); +}); + +test("rejects object input beyond the conservative nesting bound without stack overflow", async () => { + const root: Record = {}; + let cursor = root; + for (let depth = 0; depth < 10_000; depth += 1) { + const child: Record = {}; + cursor.child = child; + cursor = child; + } + await assert.rejects( + () => importSource({ sourceKind: "ci", path: "ci.json", input: { type: "object", value: root } }), + /nesting.*128/i, + ); +}); + +test("rejects file traversal and symlink escapes from the declared root", async () => { + const base = await mkdtemp(join(tmpdir(), "source-import-")); + const root = join(base, "root"); await mkdir(root); + await writeFile(join(base, "outside.py"), "import os\n"); + await symlink(join(base, "outside.py"), join(root, "escape.py")); + await assert.rejects(() => importSource({ sourceKind: "python", path: "../outside.py", input: { type: "file", root } }), /outside.*root|escape/i); + await assert.rejects(() => importSource({ sourceKind: "python", path: "escape.py", input: { type: "file", root } }), /outside.*root|symlink.*escape/i); +}); + +test("refuses even root-confined symlinks so substitution cannot change an opened source", async () => { + const root = await mkdtemp(join(tmpdir(), "source-no-follow-")); + await writeFile(join(root, "real.py"), "import safe\n"); + await symlink("real.py", join(root, "substituted.py")); + await assert.rejects( + () => importSource({ sourceKind: "python", path: "substituted.py", input: { type: "file", root } }), + /symlink|symbolic|safe source/i, + ); +}); + +test("ingests directories in deterministic path order with confined provenance", async () => { + const root = await mkdtemp(join(tmpdir(), "source-directory-")); + await mkdir(join(root, "src")); + await writeFile(join(root, "src", "b.py"), "import zlib\n"); + await writeFile(join(root, "src", "a.py"), "from pkg import x\n"); + const request = { sourceKind: "python" as const, path: "src", input: { type: "file" as const, root } }; + const first = await importSource(request); const second = await importSource(request); + assert.equal(JSON.stringify(first), JSON.stringify(second)); + assert.deepEqual(first.diagram.pages[0].nodes.filter((n) => n.kind === "module").map((n) => n.label), ["src/a.py", "src/b.py"]); + assert.deepEqual(first.diagram.pages[0].nodes.filter((n) => n.kind === "library").map((n) => n.label), ["pkg", "zlib"]); + assert.equal(first.diagram.pages[0].edges[0].provenance?.path, "src/a.py"); + assert.doesNotThrow(() => validateDiagramIR(first.diagram)); +}); + +test("reports dynamic code imports as unknown constructs instead of guessing", async () => { + const python = await textInput("python", "dynamic.py", "module = __import__(name)\n"); + const javascript = await textInput("javascript", "dynamic.js", "const module = await import(name);\n"); + assert.equal(python.diagnostics[0].code, "unknown-construct"); + assert.equal(javascript.diagnostics[0].code, "unknown-construct"); + assert.deepEqual(python.diagram.pages[0].nodes.map((n) => n.label), ["dynamic.py"]); +}); + +test("loads YAML data without permitting custom tags", async () => { + const root = await mkdtemp(join(tmpdir(), "source-yaml-")); + await writeFile(join(root, "objects.yaml"), "apiVersion: v1\nkind: Service\nmetadata:\n name: api\nspec:\n selector:\n app: api\n"); + await writeFile(join(root, "tagged.yaml"), "jobs: !exec dangerous\n"); + const result = await importSource({ sourceKind: "kubernetes", path: "objects.yaml", input: { type: "file", root } }); + assert.deepEqual(result.diagram.pages[0].nodes.map((n) => n.label), ["Service/default/api"]); + await assert.rejects(() => importSource({ sourceKind: "ci", path: "tagged.yaml", input: { type: "file", root } }), /tag|malformed YAML/i); +}); diff --git a/scripts/src/services/source-importers/index.ts b/scripts/src/services/source-importers/index.ts new file mode 100644 index 0000000..5232af9 --- /dev/null +++ b/scripts/src/services/source-importers/index.ts @@ -0,0 +1,745 @@ +import { constants } from "node:fs"; +import { open, readdir, realpath } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import { isAbsolute, relative, resolve } from "node:path"; + +import { JSON_SCHEMA, loadAll } from "js-yaml"; + +import type { DiagramEdge, DiagramIRV2, DiagramNode } from "../../model/diagram-ir.js"; +import { validateDiagramIR } from "../../model/diagram-ir.js"; + +export type SourceKind = "python" | "javascript" | "typescript" | "go" | "rust" | "terraform" | "kubernetes" | "docker-compose" | "sql" | "openapi" | "ci"; +export type SourceInput = { type: "text"; text: string } | { type: "object"; value: unknown } | { type: "file"; root: string }; +export interface SourceImportRequest { sourceKind: SourceKind; path: string; input: SourceInput; maxBytes?: number } +export interface ImportDiagnostic { code: string; severity: "warning" | "error"; message: string; path: string; line?: number } +export interface SourceImportResult { diagram: DiagramIRV2; diagnostics: ImportDiagnostic[]; provenance: { sourceKind: SourceKind; path: string } } + +function id(prefix: string, ...values: string[]): string { + const encoded = values.map((value) => Buffer.from(value, "utf8").toString("hex") || "0"); + return `${prefix}-${encoded.join(".")}`; +} + +function codePointCompare(left: string, right: string): number { + const leftPoints = [...left]; + const rightPoints = [...right]; + const length = Math.min(leftPoints.length, rightPoints.length); + for (let index = 0; index < length; index += 1) { + const difference = leftPoints[index].codePointAt(0)! - rightPoints[index].codePointAt(0)!; + if (difference !== 0) return difference; + } + return leftPoints.length - rightPoints.length; +} + +function diagramFor(request: SourceImportRequest, nodes: DiagramNode[], edges: DiagramEdge[]): DiagramIRV2 { + return validateDiagramIR({ version: 2, title: request.path, pages: [{ id: "source-import", title: request.path, nodes, edges, layout: { type: "layered", direction: "horizontal" } }], provenance: { sourceKind: request.sourceKind, path: request.path } }) as DiagramIRV2; +} + +function codeImports(kind: "python" | "go", text: string): Array<{ name: string; line: number }> { + const imports: Array<{ name: string; line: number }> = []; + let goBlock = false; + text.split(/\r?\n/).forEach((line, index) => { + let name: string | undefined; + if (kind === "python") { + const match = line.match(/^\s*(?:import\s+([A-Za-z_][\w.]*)|from\s+([A-Za-z_][\w.]*)\s+import\s+)/); + name = match?.[1] ?? match?.[2]; + } else if (kind === "go") { + if (/^\s*import\s*\(\s*$/.test(line)) { goBlock = true; return; } + if (goBlock && /^\s*\)/.test(line)) { goBlock = false; return; } + const match = goBlock ? line.match(/^\s*(?:[._A-Za-z]\w*\s+)?"([^"]+)"/) : line.match(/^\s*import\s+(?:[._A-Za-z]\w*\s+)?"([^"]+)"/); + name = match?.[1]; + } + if (name) imports.push({ name, line: index + 1 }); + }); + return imports; +} + +interface MaskedSource { text: string; unknownLines: number[] } + +function maskRange(characters: string[], start: number, end: number): void { + for (let index = start; index < end; index += 1) if (characters[index] !== "\n" && characters[index] !== "\r") characters[index] = " "; +} + +function maskPythonTripleStrings(text: string): MaskedSource { + const characters = text.split(""); + const unknownLines: number[] = []; + let offset = 0; let line = 1; + while (offset < text.length) { + if (text[offset] === "\n") { line += 1; offset += 1; continue; } + const quote = text.startsWith("'''", offset) ? "'''" : text.startsWith('"""', offset) ? '"""' : undefined; + if (!quote) { offset += 1; continue; } + const start = offset; const startLine = line; + offset += 3; + let closed = false; + while (offset < text.length) { + if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; } + if (text[offset] === "\n") line += 1; + if (text.startsWith(quote, offset)) { offset += 3; closed = true; break; } + offset += 1; + } + maskRange(characters, start, offset); + if (!closed) unknownLines.push(startLine); + } + return { text: characters.join(""), unknownLines }; +} + +function maskComments(text: string, lineMarkers: string[], nestedBlock = false, maskedQuotes = "", rustRawStrings = false): MaskedSource { + const characters = text.split(""); + const unknownLines: number[] = []; + let offset = 0; let line = 1; + while (offset < text.length) { + if (text[offset] === "\n") { line += 1; offset += 1; continue; } + const marker = lineMarkers.find((candidate) => text.startsWith(candidate, offset)); + if (marker) { + const start = offset; + while (offset < text.length && text[offset] !== "\n") offset += 1; + maskRange(characters, start, offset); + continue; + } + if (text.startsWith("/*", offset)) { + const start = offset; const startLine = line; + offset += 2; + let depth = 1; + while (offset < text.length && depth > 0) { + if (text[offset] === "\n") line += 1; + if (nestedBlock && text.startsWith("/*", offset)) { depth += 1; offset += 2; continue; } + if (text.startsWith("*/", offset)) { depth -= 1; offset += 2; continue; } + offset += 1; + } + maskRange(characters, start, offset); + if (depth > 0) unknownLines.push(startLine); + continue; + } + if (rustRawStrings && text[offset] === "r") { + let delimiterEnd = offset + 1; + while (text[delimiterEnd] === "#") delimiterEnd += 1; + if (text[delimiterEnd] === '"') { + const start = offset; const startLine = line; + const terminator = `"${"#".repeat(delimiterEnd - offset - 1)}`; + offset = delimiterEnd + 1; + const rawEnd = text.indexOf(terminator, offset); + const closed = rawEnd >= 0; + const next = closed ? rawEnd + terminator.length : text.length; + line += (text.slice(offset, next).match(/\n/g) ?? []).length; + offset = next; + maskRange(characters, start, offset); + if (!closed) unknownLines.push(startLine); + continue; + } + } + if (text[offset] === "'" || text[offset] === '"' || text[offset] === "`") { + const quote = text[offset]; const start = offset; const startLine = line; + offset += 1; + let closed = false; + while (offset < text.length) { + if (text[offset] === "\\" && quote !== "`" && offset + 1 < text.length) { offset += 2; continue; } + if (quote === "'" && text[offset] === "'" && text[offset + 1] === "'") { offset += 2; continue; } + if (text[offset] === "\n") line += 1; + if (text[offset] === quote) { offset += 1; closed = true; break; } + offset += 1; + } + if (maskedQuotes.includes(quote)) maskRange(characters, start, offset); + if (!closed) unknownLines.push(startLine); + continue; + } + offset += 1; + } + return { text: characters.join(""), unknownLines }; +} + +function maskTerraformHeredocs(text: string): MaskedSource { + const characters = text.split(""); + const unknownLines: number[] = []; + const heredocOpening = /<<(-?)([A-Za-z_]\w*)[ \t]*(?:\r?\n)/y; + let offset = 0; let line = 1; + while (offset < text.length) { + if (text[offset] === "\n") { line += 1; offset += 1; continue; } + if (text[offset] === "#" || text.startsWith("//", offset)) { + while (offset < text.length && text[offset] !== "\n") offset += 1; + continue; + } + if (text.startsWith("/*", offset)) { + const end = text.indexOf("*/", offset + 2); + const next = end < 0 ? text.length : end + 2; + line += (text.slice(offset, next).match(/\n/g) ?? []).length; + offset = next; + continue; + } + if (text[offset] === '"') { + offset += 1; + while (offset < text.length) { + if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; } + if (text[offset] === "\n") line += 1; + if (text[offset] === '"') { offset += 1; break; } + offset += 1; + } + continue; + } + heredocOpening.lastIndex = offset; + const opening = heredocOpening.exec(text); + if (!opening) { offset += 1; continue; } + const start = offset; const startLine = line; + const indented = opening[1] === "-"; + const delimiter = opening[2]; + offset += opening[0].length; + line += 1; + let closed = false; + while (offset < text.length) { + const newline = text.indexOf("\n", offset); + const end = newline < 0 ? text.length : newline; + const candidate = text.slice(offset, end).replace(/\r$/, ""); + const expected = indented ? candidate.trim() : candidate.trimEnd(); + if (expected === delimiter && (indented || candidate === expected)) { + offset = end; + closed = true; + break; + } + offset = newline < 0 ? text.length : newline + 1; + if (newline >= 0) line += 1; + } + maskRange(characters, start, offset); + if (!closed) unknownLines.push(startLine); + } + return { text: characters.join(""), unknownLines }; +} + +function maskTerraformStrings(text: string): MaskedSource { + const characters = text.split(""); + const unknownLines: number[] = []; + let offset = 0; let line = 1; + while (offset < text.length) { + if (text[offset] === "\n") { line += 1; offset += 1; continue; } + if (text[offset] !== '"') { offset += 1; continue; } + const start = offset; const startLine = line; + offset += 1; + let closed = false; + while (offset < text.length) { + if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; } + if (text[offset] === "\n") line += 1; + if (text[offset] === '"') { offset += 1; closed = true; break; } + offset += 1; + } + maskRange(characters, start, offset); + if (!closed) unknownLines.push(startLine); + } + return { text: characters.join(""), unknownLines }; +} + +interface JavaScriptToken { kind: "identifier" | "string" | "punctuation"; value: string; line: number } + +function scanJavaScriptImports(text: string): { imports: Array<{ name: string; line: number }>; unknownLines: number[] } { + const tokens: JavaScriptToken[] = []; + const unknownLines = new Set(); + let offset = 0; + let line = 1; + while (offset < text.length) { + const character = text[offset]; + if (character === "\n") { line += 1; offset += 1; continue; } + if (/\s/.test(character)) { offset += 1; continue; } + if (character === "/" && text[offset + 1] === "/") { + offset += 2; + while (offset < text.length && text[offset] !== "\n") offset += 1; + continue; + } + if (character === "/" && text[offset + 1] === "*") { + const startLine = line; + offset += 2; + let closed = false; + while (offset < text.length) { + if (text[offset] === "\n") line += 1; + if (text[offset] === "*" && text[offset + 1] === "/") { offset += 2; closed = true; break; } + offset += 1; + } + if (!closed) unknownLines.add(startLine); + continue; + } + if (character === "/") { + const previous = tokens[tokens.length - 1]; + const followsArrow = previous?.value === ">" && tokens[tokens.length - 2]?.value === "="; + const regexPrefix = !previous || followsArrow || (previous.kind === "punctuation" && "=([{,:;!&|?".includes(previous.value)) || (previous.kind === "identifier" && new Set(["return", "case", "throw", "typeof", "instanceof", "in", "of", "yield", "await"]).has(previous.value)); + if (regexPrefix) { + const startLine = line; + let inCharacterClass = false; + let closed = false; + offset += 1; + while (offset < text.length) { + if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; } + if (text[offset] === "\n") break; + if (text[offset] === "[") inCharacterClass = true; + else if (text[offset] === "]") inCharacterClass = false; + else if (text[offset] === "/" && !inCharacterClass) { offset += 1; closed = true; break; } + offset += 1; + } + while (closed && offset < text.length && /[A-Za-z]/.test(text[offset])) offset += 1; + if (!closed) unknownLines.add(startLine); + continue; + } + } + if (character === "'" || character === '"') { + const quote = character; + const startLine = line; + let value = ""; + offset += 1; + let closed = false; + while (offset < text.length) { + const child = text[offset]; + if (child === "\\" && offset + 1 < text.length) { value += child + text[offset + 1]; offset += 2; continue; } + if (child === quote) { offset += 1; closed = true; break; } + if (child === "\n") line += 1; + value += child; + offset += 1; + } + if (closed) tokens.push({ kind: "string", value, line: startLine }); + else unknownLines.add(startLine); + continue; + } + if (character === "`") { + const startLine = line; + let closed = false; + let hasExpression = false; + offset += 1; + while (offset < text.length) { + if (text[offset] === "\\" && offset + 1 < text.length) { offset += 2; continue; } + if (text[offset] === "$" && text[offset + 1] === "{") hasExpression = true; + if (text[offset] === "\n") line += 1; + if (text[offset] === "`") { offset += 1; closed = true; break; } + offset += 1; + } + if (hasExpression || !closed) unknownLines.add(startLine); + continue; + } + if (/[A-Za-z_$]/.test(character)) { + const start = offset; + offset += 1; + while (offset < text.length && /[A-Za-z0-9_$]/.test(text[offset])) offset += 1; + tokens.push({ kind: "identifier", value: text.slice(start, offset), line }); + continue; + } + tokens.push({ kind: "punctuation", value: character, line }); + offset += 1; + } + + const imports: Array<{ name: string; line: number }> = []; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + if (token.kind !== "identifier") continue; + if (token.value === "require" && tokens[index - 1]?.value !== "." && tokens[index + 1]?.value === "(") { + const argument = tokens[index + 2]; + if (argument?.kind === "string" && tokens[index + 3]?.value === ")") imports.push({ name: argument.value, line: token.line }); + else unknownLines.add(token.line); + continue; + } + if (token.value !== "import" && token.value !== "export") continue; + if (tokens[index - 1]?.value === ".") continue; + const next = tokens[index + 1]; + if (token.value === "import" && next?.value === "(") { unknownLines.add(token.line); continue; } + const previous = tokens[index - 1]; + const statementPosition = !previous || previous.value === ";" || previous.value === "}" || previous.line < token.line; + if (!statementPosition) continue; + if (token.value === "import" && next?.kind === "string") { imports.push({ name: next.value, line: token.line }); continue; } + let found = false; + for (let cursor = index + 1; cursor < tokens.length && cursor <= index + 256; cursor += 1) { + if (tokens[cursor].value === ";") break; + if (tokens[cursor].value === "from" && tokens[cursor + 1]?.kind === "string") { + imports.push({ name: tokens[cursor + 1].value, line: token.line }); + found = true; + break; + } + } + if (!found && token.value === "import") unknownLines.add(token.line); + } + return { imports, unknownLines: [...unknownLines].sort((a, b) => a - b) }; +} + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined; +} + +function objectName(value: unknown): string | undefined { + const name = record(record(value)?.metadata)?.name; + return typeof name === "string" && name.length > 0 ? name : undefined; +} + +function collectWorkloadSecretRefs(podSpecValue: unknown, refs: Set): void { + const podSpec = record(podSpecValue); + if (!podSpec) return; + const addName = (value: unknown): void => { + const name = record(value)?.name; + if (typeof name === "string" && name.length > 0) refs.add(name); + }; + for (const pullSecret of Array.isArray(podSpec.imagePullSecrets) ? podSpec.imagePullSecrets : []) addName(pullSecret); + for (const containerKey of ["initContainers", "containers", "ephemeralContainers"]) { + for (const containerValue of Array.isArray(podSpec[containerKey]) ? podSpec[containerKey] as unknown[] : []) { + const container = record(containerValue); + if (!container) continue; + for (const source of Array.isArray(container.envFrom) ? container.envFrom : []) addName(record(source)?.secretRef); + for (const environment of Array.isArray(container.env) ? container.env : []) addName(record(record(environment)?.valueFrom)?.secretKeyRef); + } + } + for (const volumeValue of Array.isArray(podSpec.volumes) ? podSpec.volumes : []) { + const volume = record(volumeValue); + const secretName = record(volume?.secret)?.secretName; + if (typeof secretName === "string" && secretName.length > 0) refs.add(secretName); + const sources = record(volume?.projected)?.sources; + for (const source of Array.isArray(sources) ? sources : []) addName(record(source)?.secret); + } +} + +function collectSchemaRefs(value: unknown, refs: Set, seen = new WeakSet()): void { + if (value === null || typeof value !== "object" || seen.has(value)) return; + seen.add(value); + if (Array.isArray(value)) { value.forEach((item) => collectSchemaRefs(item, refs, seen)); return; } + for (const [key, child] of Object.entries(value as Record)) { + if (key === "$ref" && typeof child === "string" && child.startsWith("#/components/schemas/")) refs.add(child.slice("#/components/schemas/".length)); + else collectSchemaRefs(child, refs, seen); + } +} + +function validateBoundedInput(request: SourceImportRequest): void { + const limit = request.maxBytes ?? 1_048_576; + if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 8_388_608) throw new Error("maxBytes must be an integer between 1 and 8388608"); + if (!request.path || request.path.includes("\0")) throw new Error("Source path must be a non-empty safe string"); + if (request.input.type === "file") return; + if (request.input.type === "text") { + const size = new TextEncoder().encode(request.input.text).byteLength; + if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`); + return; + } + let size = 0; + const seen = new WeakSet(); + const pending: Array<{ value: unknown; depth: number }> = [{ value: request.input.value, depth: 0 }]; + while (pending.length > 0) { + const { value, depth } = pending.pop()!; + if (depth > 128) throw new Error("Object input nesting exceeds maximum depth 128"); + if (value === null || typeof value === "boolean") size += 4; + else if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Object input must contain finite JSON values"); + size += 8; + } else if (typeof value === "string") size += new TextEncoder().encode(value).byteLength; + else { + if (typeof value !== "object") throw new Error("Object input must contain plain JSON values"); + if (seen.has(value)) throw new Error("Object input must contain acyclic plain JSON values"); + seen.add(value); + if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new Error("Object input must contain plain JSON objects"); + const descriptors = Object.getOwnPropertyDescriptors(value); + for (const [key, descriptor] of Object.entries(descriptors)) { + if (key === "length") continue; + if (!("value" in descriptor)) throw new Error("Object input must contain plain JSON data properties"); + size += new TextEncoder().encode(key).byteLength; + if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`); + pending.push({ value: descriptor.value, depth: depth + 1 }); + } + } + if (size > limit) throw new Error(`Source input exceeds maxBytes ${limit}`); + } +} + +async function importInline(request: SourceImportRequest): Promise { + validateBoundedInput(request); + if (request.sourceKind === "ci" && request.input.type === "object") { + const jobs = record(record(request.input.value)?.jobs); + if (!jobs || Object.keys(jobs).length === 0) throw new Error("Malformed CI input: jobs must be a non-empty object"); + const names = Object.keys(jobs).sort(codePointCompare); + const nodes: DiagramNode[] = names.map((name) => ({ id: id("command", name), label: name, kind: "command", provenance: { path: request.path } })); + const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = []; + for (const name of names) { + const rawNeeds = record(jobs[name])?.needs; + const needs = (Array.isArray(rawNeeds) ? rawNeeds : rawNeeds === undefined ? [] : [rawNeeds]).filter((value): value is string => typeof value === "string").sort(codePointCompare); + for (const dependency of needs) { + if (!names.includes(dependency)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown CI job: ${dependency}`, path: request.path }); continue; } + edges.push({ id: id("edge", dependency, "needs", name, String(edges.length)), source: id("command", dependency), target: id("command", name), kind: "needs", provenance: { path: request.path } }); + } + } + return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; + } + if (request.sourceKind === "openapi" && request.input.type === "object") { + const root = record(request.input.value); const paths = record(root?.paths); const schemas = record(record(root?.components)?.schemas) ?? {}; + if (!root || !paths) throw new Error("Malformed OpenAPI input: paths must be an object"); + const nodes: DiagramNode[] = []; const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = []; + const methods = new Set(["get", "put", "post", "delete", "patch", "options", "head", "trace"]); + for (const path of Object.keys(paths).sort(codePointCompare)) { + const pathItem = record(paths[path]) ?? {}; + for (const method of Object.keys(pathItem).filter((key) => methods.has(key.toLowerCase())).sort(codePointCompare)) { + const operationId = id("command", method, path); + nodes.push({ id: operationId, label: `${method.toUpperCase()} ${path}`, kind: "command", provenance: { path: request.path } }); + const refs = new Set(); collectSchemaRefs(pathItem[method], refs); + for (const schema of [...refs].sort(codePointCompare)) { + if (!Object.prototype.hasOwnProperty.call(schemas, schema)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown schema: ${schema}`, path: request.path }); continue; } + edges.push({ id: id("edge", method, path, schema, String(edges.length)), source: operationId, target: id("schema", schema), kind: "schema-reference", provenance: { path: request.path } }); + } + } + } + for (const schema of Object.keys(schemas).sort(codePointCompare)) nodes.push({ id: id("schema", schema), label: schema, kind: "library", provenance: { path: request.path } }); + return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; + } + if (request.sourceKind === "sql" && request.input.type === "text") { + const scanned = maskComments(request.input.text, ["--"], false, "'"); + const tables = new Map(); const refs: Array<{ source: string; target: string; line: number }> = []; + const diagnostics: ImportDiagnostic[] = scanned.unknownLines.map((line) => ({ code: "unknown-construct", severity: "warning", message: "SQL construct is outside the bounded static subset", path: request.path, line })); + let current: string | undefined; + scanned.text.split(/\r?\n/).forEach((line, index) => { + const declaration = line.match(/^\s*CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?["`\[]?([A-Za-z_]\w*)/i)?.[1]; + if (declaration) { + if (tables.has(declaration)) diagnostics.push({ code: "duplicate-id", severity: "warning", message: `Duplicate table: ${declaration}`, path: request.path, line: index + 1 }); + else tables.set(declaration, index + 1); + current = declaration; + } + const target = line.match(/\bREFERENCES\s+["`\[]?([A-Za-z_]\w*)/i)?.[1]; + if (current && target) refs.push({ source: current, target, line: index + 1 }); + if (/\)\s*;/.test(line)) current = undefined; + }); + const nodes: DiagramNode[] = [...tables].sort(([a], [b]) => codePointCompare(a, b)).map(([name, line]) => ({ id: id("table", name), label: name, kind: "table", provenance: { path: request.path, line } })); + const edges: DiagramEdge[] = refs.filter((ref) => tables.has(ref.target)).map((ref, occurrence) => ({ id: id("edge", ref.source, "fk", ref.target, String(occurrence)), source: id("table", ref.source), target: id("table", ref.target), kind: "foreign-key", provenance: { path: request.path, line: ref.line } })); + return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; + } + if (request.sourceKind === "docker-compose" && request.input.type === "object") { + const root = record(request.input.value); + const services = record(root?.services); + const volumes = record(root?.volumes) ?? {}; + if (!services || Object.keys(services).length === 0) throw new Error("Malformed Docker Compose input: services must be a non-empty object"); + const serviceNames = Object.keys(services).sort(codePointCompare); const volumeNames = Object.keys(volumes).sort(codePointCompare); + const nodes: DiagramNode[] = [ + ...serviceNames.map((name) => ({ id: id("service", name), label: name, kind: "service", provenance: { path: request.path } })), + ...volumeNames.map((name) => ({ id: id("volume", name), label: name, kind: "volume", provenance: { path: request.path } })), + ]; + const edges: DiagramEdge[] = []; const diagnostics: ImportDiagnostic[] = []; + for (const name of serviceNames) { + const service = record(services[name]) ?? {}; + const depends = Array.isArray(service.depends_on) ? service.depends_on : Object.keys(record(service.depends_on) ?? {}); + for (const dependency of depends.filter((value): value is string => typeof value === "string").sort(codePointCompare)) { + if (!serviceNames.includes(dependency)) { diagnostics.push({ code: "unknown-reference", severity: "warning", message: `Unknown service: ${dependency}`, path: request.path }); continue; } + edges.push({ id: id("edge", name, "depends", dependency, String(edges.length)), source: id("service", name), target: id("service", dependency), kind: "depends-on", provenance: { path: request.path } }); + } + for (const mount of (Array.isArray(service.volumes) ? service.volumes : []).filter((value): value is string => typeof value === "string").sort(codePointCompare)) { + const volume = mount.split(":", 1)[0]; + if (volumeNames.includes(volume)) edges.push({ id: id("edge", name, "mounts", volume, String(edges.length)), source: id("service", name), target: id("volume", volume), kind: "mounts", provenance: { path: request.path } }); + } + } + return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; + } + if (request.sourceKind === "kubernetes" && request.input.type === "object") { + const values = Array.isArray(request.input.value) ? request.input.value : [request.input.value]; + const rawObjects = values.map(record).filter((value): value is Record => Boolean(value && typeof value.kind === "string" && objectName(value))); + const occurrences = new Map(); + const objects = rawObjects.map((value) => { + const kind = String(value.kind); + const name = objectName(value)!; + const rawNamespace = record(value.metadata)?.namespace; + const namespace = typeof rawNamespace === "string" && rawNamespace.length > 0 ? rawNamespace : "default"; + const key = JSON.stringify([kind, namespace, name]); + const occurrence = occurrences.get(key) ?? 0; + occurrences.set(key, occurrence + 1); + return { value, kind, name, namespace, nodeId: id("k8s", kind, namespace, name, String(occurrence)) }; + }); + const diagnostics: ImportDiagnostic[] = objects.filter((object) => object.kind === "Secret").map(() => ({ code: "secret-redacted", severity: "warning", message: "Kubernetes Secret payload omitted", path: request.path })); + const nodes: DiagramNode[] = objects.map((object) => ({ + id: object.nodeId, + label: `${object.kind}/${object.namespace}/${object.name}`, + kind: object.kind.toLowerCase(), + provenance: { path: request.path }, + })).sort((a, b) => codePointCompare(a.label, b.label)); + const byKey = new Map(); + for (const object of objects) { + const key = JSON.stringify([object.kind, object.namespace, object.name]); + if (!byKey.has(key)) byKey.set(key, object); + } + const workloadKinds = new Set(["Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "Job"]); + const edges: DiagramEdge[] = []; + for (const object of objects) { + const secretRefs = new Set(); + if (workloadKinds.has(object.kind)) collectWorkloadSecretRefs(record(record(object.value.spec)?.template)?.spec, secretRefs); + for (const target of [...secretRefs].sort(codePointCompare)) { + const secret = byKey.get(JSON.stringify(["Secret", object.namespace, target])); + if (secret) edges.push({ id: id("edge", object.nodeId, "secret", secret.nodeId, String(edges.length)), source: object.nodeId, target: secret.nodeId, kind: "uses-secret", provenance: { path: request.path } }); + } + if (object.kind === "Service") { + const selector = record(record(object.value.spec)?.selector); + if (!selector || Object.keys(selector).length === 0) continue; + for (const workload of objects.filter((candidate) => workloadKinds.has(candidate.kind) && candidate.namespace === object.namespace)) { + const labels = record(record(record(record(workload.value.spec)?.template)?.metadata)?.labels); + if (labels && Object.entries(selector).every(([key, selected]) => labels[key] === selected)) { + edges.push({ id: id("edge", object.nodeId, "selects", workload.nodeId, String(edges.length)), source: object.nodeId, target: workload.nodeId, kind: "selects", provenance: { path: request.path } }); + } + } + } + } + return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; + } + if (request.sourceKind === "terraform" && request.input.type === "text") { + const heredocs = maskTerraformHeredocs(request.input.text); + const structural = maskComments(heredocs.text, ["#", "//"]); + const referencesOnly = maskTerraformStrings(structural.text); + const unknownLines = [...new Set([...heredocs.unknownLines, ...structural.unknownLines, ...referencesOnly.unknownLines])].sort((a, b) => a - b); + const diagnostics: ImportDiagnostic[] = unknownLines.map((line) => ({ code: "unknown-construct", severity: "warning", message: "Terraform construct is outside the bounded static subset", path: request.path, line })); + const resources = new Map(); + const references: Array<{ source: string; target: string; line: number }> = []; + const referenceLines = referencesOnly.text.split(/\r?\n/); + let current: string | undefined; + structural.text.split(/\r?\n/).forEach((line, index) => { + const declaration = line.match(/^\s*resource\s+"([^"]+)"\s+"([^"]+)"\s*\{/); + if (declaration) { + current = `${declaration[1]}.${declaration[2]}`; + resources.set(current, index + 1); + if (/\{\s*\}/.test(line)) current = undefined; + return; + } + if (current) { + for (const match of referenceLines[index].matchAll(/\b([A-Za-z_]\w*\.[A-Za-z_]\w*)\.[A-Za-z_]\w*/g)) references.push({ source: current, target: match[1], line: index + 1 }); + if (/^\s*}/.test(line)) current = undefined; + } + }); + const nodes = [...resources].sort(([a], [b]) => codePointCompare(a, b)).map(([name, line]) => ({ id: id("resource", name), label: name, kind: "resource", provenance: { path: request.path, line } })); + const edges = references.filter((ref) => resources.has(ref.target)).sort((a, b) => codePointCompare(a.source, b.source) || codePointCompare(a.target, b.target) || a.line - b.line).map((ref, occurrence) => ({ id: id("edge", ref.source, ref.target, String(occurrence)), source: id("resource", ref.source), target: id("resource", ref.target), kind: "reference", provenance: { path: request.path, line: ref.line } })); + return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; + } + if (!(["python", "javascript", "typescript", "go", "rust"] as SourceKind[]).includes(request.sourceKind) || request.input.type !== "text") throw new Error("Unsupported source input"); + const nodes: DiagramNode[] = [{ id: id("module", request.path), label: request.path, kind: "module", provenance: { path: request.path, line: 1 } }]; + const edges: DiagramEdge[] = []; + const diagnostics: ImportDiagnostic[] = []; + const javaScriptScan = request.sourceKind === "javascript" || request.sourceKind === "typescript" ? scanJavaScriptImports(request.input.text) : undefined; + const nonJavaScriptScan = request.sourceKind === "python" ? maskPythonTripleStrings(request.input.text) + : request.sourceKind === "go" ? maskComments(request.input.text, ["//"], false, "`") + : request.sourceKind === "rust" ? maskComments(request.input.text, ["//"], true, "", true) + : { text: request.input.text, unknownLines: [] }; + for (const line of javaScriptScan?.unknownLines ?? nonJavaScriptScan.unknownLines) diagnostics.push({ code: "unknown-construct", severity: "warning", message: `${request.sourceKind} construct is outside the bounded static-import subset`, path: request.path, line }); + nonJavaScriptScan.text.split(/\r?\n/).forEach((line, index) => { + const dynamicPython = request.sourceKind === "python" && /\b(?:__import__|importlib\.import_module)\s*\(\s*[^"']/.test(line); + if (dynamicPython) diagnostics.push({ code: "unknown-construct", severity: "warning", message: "Dynamic imports are not evaluated", path: request.path, line: index + 1 }); + }); + const seen = new Set(); + const foundImports = request.sourceKind === "rust" + ? nonJavaScriptScan.text.split(/\r?\n/).flatMap((line, index) => { + const mod = line.match(/^\s*mod\s+([A-Za-z_]\w*)\s*;/)?.[1]; + const use = line.match(/^\s*use\s+([^;]+);/)?.[1]; + if (/\w+!\s*\(/.test(line)) diagnostics.push({ code: "unknown-construct", severity: "warning", message: "Rust macros are not expanded", path: request.path, line: index + 1 }); + if (mod) return [{ name: mod, line: index + 1 }]; + if (use) { + const root = use.includes("::{") ? use.slice(0, use.indexOf("::{")) : use.split("::").slice(0, -1).join("::") || use; + return [{ name: root, line: index + 1 }]; + } + return []; + }) + : javaScriptScan?.imports ?? codeImports(request.sourceKind as "python" | "go", nonJavaScriptScan.text); + for (const found of foundImports) { + if (seen.has(found.name)) continue; + seen.add(found.name); + const targetId = id("library", found.name); + nodes.push({ id: targetId, label: found.name, kind: "library", provenance: { path: request.path, line: found.line } }); + edges.push({ id: id("edge", request.path, found.name, String(edges.length)), source: nodes[0].id, target: targetId, kind: "import", provenance: { path: request.path, line: found.line } }); + } + return { diagram: diagramFor(request, nodes, edges), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; +} + +function isConfined(root: string, candidate: string): boolean { + const pathFromRoot = relative(root, candidate); + return pathFromRoot === "" || (!pathFromRoot.startsWith("..") && !isAbsolute(pathFromRoot)); +} + +async function openConfined(candidate: string, root: string): Promise<{ handle: FileHandle; canonical: string }> { + let handle: FileHandle; + try { + handle = await open(candidate, constants.O_RDONLY | constants.O_NOFOLLOW); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ELOOP") throw new Error("Source symlink escape risk cannot be opened safely"); + throw error; + } + try { + const canonical = await realpath(`/proc/self/fd/${handle.fd}`); + if (!isConfined(root, canonical)) throw new Error("Opened source escapes outside declared root"); + return { handle, canonical }; + } catch (error) { + await handle.close(); + throw error; + } +} + +async function readBounded(handle: FileHandle, limit: number, directory: boolean): Promise { + const initial = await handle.stat(); + if (!initial.isFile()) throw new Error("Source entry must be a regular file"); + if (initial.size > limit) throw new Error(`${directory ? "Directory source input" : "Source input"} exceeds maxBytes ${limit}`); + const chunks: Buffer[] = []; + let consumed = 0; + while (consumed < limit) { + const chunk = Buffer.allocUnsafe(Math.min(65_536, limit - consumed)); + const { bytesRead } = await handle.read(chunk, 0, chunk.byteLength, null); + if (bytesRead === 0) break; + chunks.push(chunk.subarray(0, bytesRead)); + consumed += bytesRead; + } + const final = await handle.stat(); + if (initial.dev !== final.dev || initial.ino !== final.ino) throw new Error("Opened source identity changed during read"); + if (final.size > limit || consumed < initial.size) throw new Error(`${directory ? "Directory source input" : "Source input"} exceeds maxBytes ${limit}`); + return Buffer.concat(chunks, consumed); +} + +export async function importSource(request: SourceImportRequest): Promise { + validateBoundedInput(request); + if (request.input.type !== "file") return importInline(request); + if (isAbsolute(request.path)) throw new Error("Source path escapes outside declared root"); + const rootPath = await realpath(request.input.root); + const lexicalPath = resolve(rootPath, request.path); + if (!isConfined(rootPath, lexicalPath)) throw new Error("Source path escapes outside declared root"); + const limit = request.maxBytes ?? 1_048_576; + const objectKinds = new Set(["kubernetes", "docker-compose", "openapi", "ci"]); + const extensions: Record> = { + python: new Set([".py"]), javascript: new Set([".js", ".jsx", ".mjs", ".cjs"]), typescript: new Set([".ts", ".tsx", ".mts", ".cts"]), + go: new Set([".go"]), rust: new Set([".rs"]), terraform: new Set([".tf"]), sql: new Set([".sql"]), + kubernetes: new Set([".json", ".yaml", ".yml"]), "docker-compose": new Set([".json", ".yaml", ".yml"]), openapi: new Set([".json", ".yaml", ".yml"]), ci: new Set([".json", ".yaml", ".yml"]), + }; + const parse = async (buffer: Buffer, provenancePath: string): Promise => { + const text = new TextDecoder("utf-8", { fatal: true }).decode(buffer); + let input: SourceInput; + if (objectKinds.has(request.sourceKind)) { + const yaml = /\.ya?ml$/i.test(provenancePath); + try { + if (yaml) { + const documents = loadAll(text, undefined, { schema: JSON_SCHEMA, json: false }).filter((value) => value !== undefined); + if (documents.length === 0) throw new Error("empty"); + if (request.sourceKind !== "kubernetes" && documents.length !== 1) throw new Error("multiple documents"); + input = { type: "object", value: request.sourceKind === "kubernetes" ? documents : documents[0] }; + } else input = { type: "object", value: JSON.parse(text) as unknown }; + } catch { throw new Error(`Malformed ${yaml ? "YAML or disallowed tag" : "JSON"} source: ${provenancePath}`); } + } else input = { type: "text", text }; + return importInline({ ...request, path: provenancePath, input }); + }; + const source = await openConfined(lexicalPath, rootPath); + const info = await source.handle.stat(); + if (info.isFile()) { + try { return await parse(await readBounded(source.handle, limit, false), request.path); } + finally { await source.handle.close(); } + } + if (!info.isDirectory()) { await source.handle.close(); throw new Error("Source path must be a file or directory"); } + + const results: SourceImportResult[] = []; + let consumed = 0; + const walk = async (directory: FileHandle): Promise => { + for (const name of (await readdir(`/proc/self/fd/${directory.fd}`)).sort(codePointCompare)) { + const opened = await openConfined(resolve(`/proc/self/fd/${directory.fd}`, name), rootPath); + try { + const entryInfo = await opened.handle.stat(); + if (entryInfo.isDirectory()) await walk(opened.handle); + else if (entryInfo.isFile()) { + const dot = name.lastIndexOf("."); const extension = dot >= 0 ? name.slice(dot).toLowerCase() : ""; + if (extensions[request.sourceKind].has(extension)) { + const provenancePath = relative(rootPath, opened.canonical).split("\\").join("/"); + const buffer = await readBounded(opened.handle, limit - consumed, true); + consumed += buffer.byteLength; + results.push(await parse(buffer, provenancePath)); + } + } else throw new Error("Source directory contains an unsupported entry type"); + } finally { await opened.handle.close(); } + } + }; + try { await walk(source.handle); } + finally { await source.handle.close(); } + if (results.length === 0) throw new Error(`No supported ${request.sourceKind} source files found`); + const nodeById = new Map(); const edgeById = new Map(); const diagnostics: ImportDiagnostic[] = []; + for (const result of results) { + diagnostics.push(...result.diagnostics); + for (const node of result.diagram.pages[0].nodes) if (!nodeById.has(node.id)) nodeById.set(node.id, node); + for (const edge of result.diagram.pages[0].edges) { + if (edgeById.has(edge.id)) diagnostics.push({ code: "duplicate-id", severity: "warning", message: `Duplicate edge omitted: ${edge.id}`, path: String(edge.provenance?.path ?? request.path) }); + else edgeById.set(edge.id, edge); + } + } + return { diagram: diagramFor(request, [...nodeById.values()], [...edgeById.values()]), diagnostics, provenance: { sourceKind: request.sourceKind, path: request.path } }; +}