feat(explore): markdown index (#361) with a section-first doc tier

Ports QingNagi/codegraph#361 (markdown extractor, heading nodes, name-matcher and
resolution hooks) onto experimental and adds the section-first doc tier from
feature/md-section-first: a doc-shaped query renders the best headed sections of
the markdown file it names, ranked by idf-weighted line hits with path tokens
weighted zero, capped at DOC_FILE_CAP per file. Markdown reaches an answer only
through that tier, generated-file detection ignores markdown bodies, and the
budget tiers count code files only so a README-heavy repo keeps its code answers.

Author: bompus

__tests__/extraction.test.ts

@@ -102,6 +102,12 @@ describe('Language Detection', () => {
     expect(detectLanguage('stdio.h', '#ifndef STDIO_H\nvoid printf();\n#endif\n')).toBe('c');
   });
 
+  it('should detect Markdown files', () => {
+    expect(detectLanguage('README.md')).toBe('markdown');
+    expect(detectLanguage('docs/guide.markdown')).toBe('markdown');
+    expect(detectLanguage('docs/page.mdx')).toBe('markdown');
+  });
+
   it('should detect Metal shader files as C++ (#1121)', () => {
     expect(detectLanguage('Shaders.metal')).toBe('cpp');
     expect(isSourceFile('Renderer/Shaders.metal')).toBe(true);
@@ -250,11 +256,218 @@ describe('Language Support', () => {
     expect(languages).toContain('swift');
     expect(languages).toContain('kotlin');
     expect(languages).toContain('dart');
+    expect(languages).toContain('markdown');
     expect(languages).toContain('solidity');
     expect(languages).toContain('nix');
   });
 });
 
+describe('Markdown Extraction', () => {
+  it('should extract headings, links, and shell script references', () => {
+    const markdown = `# Project Guide
+
+See [Setup](docs/setup.md#install) and scripts/release.mjs.
+
+## Release
+
+\`\`\`bash
+npm run build
+node scripts/release.mjs
+\`\`\`
+`;
+
+    const result = extractFromSource('README.md', markdown);
+
+    const fileNode = result.nodes.find((n) => n.kind === 'file');
+    expect(fileNode).toMatchObject({
+      name: 'README.md',
+      language: 'markdown',
+    });
+
+    const headings = result.nodes.filter((n) => n.kind === 'module');
+    expect(headings.map((n) => n.name)).toContain('Project Guide');
+    expect(headings.map((n) => n.name)).toContain('Release');
+
+    const commandNode = result.nodes.find((n) => n.kind === 'function' && n.signature === 'node scripts/release.mjs');
+    expect(commandNode).toBeDefined();
+
+    expect(result.unresolvedReferences).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          referenceName: 'docs/setup.md#install',
+          referenceKind: 'imports',
+          language: 'markdown',
+        }),
+        expect.objectContaining({
+          referenceName: 'scripts/release.mjs',
+          referenceKind: 'calls',
+          language: 'markdown',
+        }),
+      ])
+    );
+  });
+
+  it('should extract structured table rows and file-symbol references from Markdown', () => {
+    const markdown = `# Maintenance Guide
+
+## Phase 4
+
+| Template | CLI Entry | Dispatcher | Implementation |
+| --- | --- | --- | --- |
+| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` |
+| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` |
+
+- P4-FLOW changes must inspect \`scripts/csv_search.py::run_p4\`.
+`;
+
+    const result = extractFromSource('phases/phase4.md', markdown);
+
+    const tableRows = result.nodes.filter((n) => n.kind === 'constant' && n.qualifiedName.includes('table-row'));
+    expect(tableRows.map((n) => n.name)).toEqual(expect.arrayContaining(['P4-S1', 'P4-S2']));
+
+    const p4s1 = tableRows.find((n) => n.name === 'P4-S1');
+    expect(p4s1?.signature).toContain('Template: P4-S1');
+    expect(p4s1?.signature).toContain('Dispatcher: scripts/csv_search.py::run_p4');
+
+    const commandNode = result.nodes.find((n) =>
+      n.kind === 'function' &&
+      n.language === 'markdown' &&
+      n.signature?.includes('python "{script_path}" p4')
+    );
+    expect(commandNode).toBeDefined();
+
+    expect(result.unresolvedReferences).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          referenceName: 'phases/scripts/csv_search.py::run_p4',
+          referenceKind: 'references',
+          language: 'markdown',
+        }),
+        expect.objectContaining({
+          referenceName: 'phases/scripts/csv_search.py::_p4_stage1',
+          referenceKind: 'references',
+          language: 'markdown',
+        }),
+      ])
+    );
+  });
+
+  it('should keep structured blocks after fences containing a different fence marker', () => {
+    const markdown = `# Runbook
+
+\`\`\`text
+~~~~
+\`\`\`
+
+- POST-FENCE references \`src/auth.ts::login\`.
+
+| Key | Target |
+| --- | --- |
+| POST-TABLE | \`src/auth.ts::login\` |
+`;
+
+    const result = extractFromSource('docs/runbook.md', markdown);
+    const constants = result.nodes.filter((n) => n.kind === 'constant');
+
+    expect(constants).toEqual(expect.arrayContaining([
+      expect.objectContaining({ docstring: 'POST-FENCE references src/auth.ts::login.' }),
+      expect.objectContaining({ name: 'POST-TABLE' }),
+    ]));
+  });
+
+  it('indexes Setext (underline) headings and skips frontmatter / code fences', () => {
+    const markdown = `---
+title: Config Doc
+---
+
+Architecture Overview
+=====================
+
+Intro paragraph for the overview.
+
+Routing Layer
+-------------
+
+\`\`\`md
+Not A Heading
+=============
+\`\`\`
+`;
+
+    const result = extractFromSource('docs/arch.md', markdown);
+    const headings = result.nodes.filter((n) => n.kind === 'module');
+    const byName = new Map(headings.map((h) => [h.name, h]));
+
+    // Setext H1 (===) and H2 (---) become module nodes.
+    expect(byName.get('Architecture Overview')?.signature).toBe('# Architecture Overview');
+    expect(byName.get('Routing Layer')?.signature).toBe('## Routing Layer');
+    // Frontmatter `title:` (above the closing `---`) is NOT a heading, and a
+    // setext-looking line inside a code fence is ignored.
+    expect(byName.has('title: Config Doc')).toBe(false);
+    expect(byName.has('Not A Heading')).toBe(false);
+  });
+
+  it('builds a deterministic, compact file digest (intro + key references)', () => {
+    const markdown = `# Release Runbook
+
+This runbook explains how to cut a release.
+
+See [setup](docs/setup.md#install) and run \`scripts/release.mjs\`.
+It dispatches \`scripts/csv_search.py::run_p4\`.
+`;
+
+    const result = extractFromSource('RUNBOOK.md', markdown);
+    const fileNode = result.nodes.find((n) => n.kind === 'file');
+
+    expect(fileNode?.docstring).toBeDefined();
+    const digest = fileNode!.docstring!;
+    // Intro is the first prose line, not the heading or a link blob.
+    expect(digest).toContain('This runbook explains how to cut a release.');
+    // Key referenced files/symbols are surfaced, compacted to basenames.
+    expect(digest).toContain('refs:');
+    expect(digest).toContain('setup.md#install');
+    expect(digest).toContain('release.mjs');
+    expect(digest).toContain('csv_search.py::run_p4');
+    // Short enough to show in node details (the < 200 char detail gate).
+    expect(digest.length).toBeLessThan(200);
+  });
+});
+
+describe('Code to Markdown Reference Extraction', () => {
+  it('should extract Markdown path references from code string literals', () => {
+    const code = `
+export const GUIDE = '../docs/guide.md';
+
+export function loadDocs() {
+  return fs.readFileSync('../docs/guide.md#install', 'utf8');
+}
+`;
+
+    const result = extractFromSource('src/load-docs.ts', code);
+    const loadDocs = result.nodes.find((n) => n.kind === 'function' && n.name === 'loadDocs');
+    const guideConstant = result.nodes.find((n) => n.kind === 'constant' && n.name === 'GUIDE');
+
+    expect(loadDocs).toBeDefined();
+    expect(guideConstant).toBeDefined();
+    expect(result.unresolvedReferences).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          fromNodeId: loadDocs!.id,
+          referenceName: 'docs/guide.md#install',
+          referenceKind: 'references',
+          language: 'typescript',
+        }),
+        expect.objectContaining({
+          fromNodeId: guideConstant!.id,
+          referenceName: 'docs/guide.md',
+          referenceKind: 'references',
+          language: 'typescript',
+        }),
+      ])
+    );
+  });
+});
+
 describe('Nix Extraction', () => {
   it('should distinguish Nix variable and function bindings', () => {
     const code = `

__tests__/integration/full-pipeline.test.ts

@@ -82,6 +82,97 @@ describe('Integration: full pipeline', () => {
     cleanupTempDir(tempDir);
   });
 
+  it('indexes Markdown headings and resolves Markdown links to script files', async () => {
+    fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true });
+    fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true });
+    fs.writeFileSync(
+      path.join(tempDir, 'README.md'),
+      `# Project Guide
+
+See [Setup](docs/setup.md#install).
+
+## Release
+
+\`\`\`bash
+node scripts/release.mjs
+\`\`\`
+`
+    );
+    fs.writeFileSync(path.join(tempDir, 'docs', 'setup.md'), '# Install\n');
+    fs.writeFileSync(path.join(tempDir, 'scripts', 'release.mjs'), 'export function release() { return true; }\n');
+
+    const cg = await CodeGraph.init(tempDir);
+    try {
+      await cg.indexAll();
+
+      const guide = cg.searchNodes('Project Guide').find((r) => r.node.language === 'markdown');
+      expect(guide).toBeDefined();
+
+      const releaseCommand = cg
+        .searchNodes('release.mjs')
+        .find((r) => r.node.language === 'markdown' && r.node.kind === 'function');
+      expect(releaseCommand).toBeDefined();
+
+      const guideEdges = cg.getOutgoingEdges(guide!.node.id).filter((e) => e.kind === 'imports');
+      const guideTargets = guideEdges.map((e) => cg.getNode(e.target));
+      const setupHeading = guideTargets.find((n) => n?.qualifiedName === 'docs/setup.md#install');
+      expect(setupHeading).toMatchObject({
+        kind: 'module',
+        name: 'Install',
+        filePath: 'docs/setup.md',
+        startLine: 1,
+      });
+
+      const commandEdges = cg.getOutgoingEdges(releaseCommand!.node.id).filter((e) => e.kind === 'calls');
+      const commandTargets = commandEdges.map((e) => cg.getNode(e.target)?.filePath);
+      expect(commandTargets).toContain('scripts/release.mjs');
+    } finally {
+      cg.destroy();
+    }
+  });
+
+  it('indexes Markdown template tables and resolves file-symbol references to implementation functions', async () => {
+    fs.mkdirSync(path.join(tempDir, 'phases'), { recursive: true });
+    fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true });
+    fs.writeFileSync(
+      path.join(tempDir, 'phases', 'phase4.md'),
+      `# Phase 4
+
+## Fixed Script Templates
+
+| Template | CLI Entry | Dispatcher | Implementation |
+| --- | --- | --- | --- |
+| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` |
+| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` |
+`
+    );
+    fs.writeFileSync(
+      path.join(tempDir, 'scripts', 'csv_search.py'),
+      `def _p4_stage1(filepath, condition_spec, probe_cols_spec):\n    return 's1'\n\n` +
+        `def _p4_stage2(filepath, stage1_rows, condition_spec, detail_cols_spec):\n    return 's2'\n\n` +
+        `def run_p4(filepath, args):\n    return _p4_stage1(filepath, '-', 'MPN')\n`
+    );
+
+    const cg = await CodeGraph.init(tempDir);
+    try {
+      await cg.indexAll();
+
+      const p4s1Row = cg.searchNodes('P4-S1').find((r) => r.node.language === 'markdown');
+      expect(p4s1Row?.node.kind).toBe('constant');
+
+      const edges = cg.getOutgoingEdges(p4s1Row!.node.id).filter((e) => e.kind === 'references');
+      const targets = edges.map((e) => cg.getNode(e.target));
+      expect(targets).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ name: 'run_p4', filePath: 'scripts/csv_search.py' }),
+          expect.objectContaining({ name: '_p4_stage1', filePath: 'scripts/csv_search.py' }),
+        ])
+      );
+    } finally {
+      cg.destroy();
+    }
+  });
+
   it('runs init → index → resolve → search → callers → context → sync', async () => {
     const MODULE_COUNT = 120;
     generateSyntheticProject(tempDir, MODULE_COUNT);
@@ -269,4 +360,46 @@ describe('Integration: full pipeline', () => {
       cg.destroy();
     }
   }, 30_000);
+
+  it('resolves code string references to Markdown headings', async () => {
+    fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true });
+    fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true });
+    fs.writeFileSync(
+      path.join(tempDir, 'docs', 'guide.md'),
+      `# Guide
+
+## Install
+
+Run the setup command.
+`
+    );
+    fs.writeFileSync(
+      path.join(tempDir, 'scripts', 'load_docs.py'),
+      `GUIDE = "docs/guide.md"\n\n` +
+        `def load_docs():\n` +
+        `    return open("docs/guide.md#install", encoding="utf-8").read()\n`
+    );
+
+    const cg = await CodeGraph.init(tempDir);
+    try {
+      await cg.indexAll();
+
+      const loadDocs = cg.searchNodes('load_docs').find((r) => r.node.language === 'python');
+      expect(loadDocs).toBeDefined();
+
+      const edges = cg.getOutgoingEdges(loadDocs!.node.id).filter((e) => e.kind === 'references');
+      const targets = edges.map((e) => cg.getNode(e.target));
+      expect(targets).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            kind: 'module',
+            name: 'Install',
+            qualifiedName: 'docs/guide.md#install',
+          }),
+        ])
+      );
+    } finally {
+      cg.destroy();
+    }
+  });
 });