mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-11-16 12:05:12 -06:00
354 lines
12 KiB
JavaScript
354 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* sync-taxonomy.mjs
|
|
*
|
|
* Reads the canonical Machine Learning Systems Taxonomy from
|
|
* `taxonomy/framework-data.json` and generates `socratiq/src_shared/taxonomy-map.js`
|
|
* with zero runtime dependencies.
|
|
*/
|
|
|
|
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
|
import { dirname, resolve } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const REPO_ROOT = resolve(__dirname, '../..');
|
|
const FRAMEWORK_JSON_PATH = resolve(REPO_ROOT, 'taxonomy/framework-data.json');
|
|
const OUTPUT_PATH = resolve(__dirname, '../src_shared/taxonomy-map.js');
|
|
|
|
const GROUP_COLORS = {
|
|
F: '#2563eb',
|
|
S: '#7c3aed',
|
|
P: '#059669',
|
|
Q: '#d97706',
|
|
T: '#dc2626',
|
|
};
|
|
|
|
const GROUP_ORDER = ['F', 'S', 'P', 'Q', 'T'];
|
|
|
|
async function main() {
|
|
const raw = await readFile(FRAMEWORK_JSON_PATH, 'utf8');
|
|
const data = JSON.parse(raw);
|
|
const framework = data.framework || {};
|
|
|
|
// 1. Build TAXONOMY_GROUPS
|
|
const groupsMap = {};
|
|
for (const group of framework.groups || []) {
|
|
const code = String(group.code || '').trim();
|
|
if (!code) continue;
|
|
groupsMap[code] = {
|
|
id: group.id,
|
|
code,
|
|
name: group.label,
|
|
question: group.question,
|
|
color: GROUP_COLORS[code] || '#2563eb',
|
|
};
|
|
}
|
|
|
|
// 2. Build TAXONOMY_SUBJECTS & collect chapter associations
|
|
const subjectsMap = {};
|
|
const chapterIndexAccumulator = new Map();
|
|
|
|
function ensureChapter(slug, volume, title) {
|
|
if (!slug) return null;
|
|
if (!chapterIndexAccumulator.has(slug)) {
|
|
chapterIndexAccumulator.set(slug, {
|
|
slug,
|
|
volume: volume || slug.split('/')[0] || 'vol1',
|
|
title: title || slug,
|
|
groupWeights: { F: 0, S: 0, P: 0, Q: 0, T: 0 },
|
|
groupsSet: new Set(),
|
|
directSubjects: new Set(),
|
|
linkedSubjects: new Set(),
|
|
});
|
|
} else {
|
|
const existing = chapterIndexAccumulator.get(slug);
|
|
if (title && (!existing.title || existing.title === slug)) {
|
|
existing.title = title;
|
|
}
|
|
if (volume && !existing.volume) {
|
|
existing.volume = volume;
|
|
}
|
|
}
|
|
return chapterIndexAccumulator.get(slug);
|
|
}
|
|
|
|
for (const area of framework.areas || []) {
|
|
const areaCode = String(area.code || '').trim();
|
|
const areaName = area.display_label || area.label || areaCode;
|
|
const groupCode = areaCode.charAt(0);
|
|
|
|
for (const row of area.rows || []) {
|
|
const code = String(row.code || '').trim();
|
|
if (!code) continue;
|
|
const name = row.display_label || row.label || code;
|
|
const fullLabel = row.label && row.label !== name ? ` (${row.label})` : '';
|
|
const includesText = Array.isArray(row.includes) && row.includes.length > 0
|
|
? ` Includes: ${row.includes.join('; ')}.`
|
|
: '';
|
|
const summary = `${row.definition || ''}${fullLabel}${includesText}`.trim();
|
|
|
|
const chapterSlugs = new Set();
|
|
for (const ch of row.book_chapters || []) {
|
|
if (!ch?.chapter) continue;
|
|
chapterSlugs.add(ch.chapter);
|
|
const entry = ensureChapter(ch.chapter, ch.volume, ch.title);
|
|
if (entry) {
|
|
entry.groupWeights[groupCode] = (entry.groupWeights[groupCode] || 0) + 3;
|
|
entry.groupsSet.add(groupCode);
|
|
entry.directSubjects.add(code);
|
|
}
|
|
}
|
|
|
|
for (const bl of row.book_links || []) {
|
|
if (!bl?.chapter) continue;
|
|
chapterSlugs.add(bl.chapter);
|
|
const entry = ensureChapter(bl.chapter, bl.volume, bl.chapter_title);
|
|
if (entry) {
|
|
entry.groupWeights[groupCode] = (entry.groupWeights[groupCode] || 0) + 1;
|
|
entry.groupsSet.add(groupCode);
|
|
entry.linkedSubjects.add(code);
|
|
}
|
|
}
|
|
|
|
subjectsMap[code] = {
|
|
code,
|
|
name,
|
|
group: groupCode,
|
|
areaCode,
|
|
areaName,
|
|
summary,
|
|
chapters: [...chapterSlugs],
|
|
};
|
|
}
|
|
}
|
|
|
|
// Also ensure any top-level book_links chapters are present
|
|
for (const bl of data.book_links || []) {
|
|
if (bl?.chapter) {
|
|
ensureChapter(bl.chapter, bl.volume, bl.chapter_title);
|
|
}
|
|
}
|
|
|
|
// 3. Build CHAPTER_TAXONOMY_INDEX sorted by chapter slug
|
|
const chapterTaxonomyIndex = {};
|
|
const sortedSlugs = [...chapterIndexAccumulator.keys()].sort();
|
|
for (const slug of sortedSlugs) {
|
|
const item = chapterIndexAccumulator.get(slug);
|
|
const groups = GROUP_ORDER.filter((g) => item.groupsSet.has(g));
|
|
const primaryGroup = GROUP_ORDER.slice().sort(
|
|
(a, b) => (item.groupWeights[b] || 0) - (item.groupWeights[a] || 0)
|
|
)[0] || groups[0] || 'F';
|
|
const subjects = [
|
|
...item.directSubjects,
|
|
...[...item.linkedSubjects].filter((code) => !item.directSubjects.has(code)),
|
|
];
|
|
|
|
chapterTaxonomyIndex[slug] = {
|
|
slug: item.slug,
|
|
volume: item.volume,
|
|
title: item.title,
|
|
primaryGroup,
|
|
groups: groups.length ? groups : [primaryGroup],
|
|
subjects,
|
|
};
|
|
}
|
|
|
|
const moduleSource = `// AUTO-GENERATED by socratiq/scripts/sync-taxonomy.mjs from taxonomy/framework-data.json
|
|
// Do not hand-edit without re-running: node scripts/sync-taxonomy.mjs
|
|
|
|
function attachLookupHelpers(record) {
|
|
Object.defineProperties(record, {
|
|
get: { value: (key) => record[key], enumerable: false },
|
|
has: { value: (key) => Object.prototype.hasOwnProperty.call(record, key), enumerable: false },
|
|
size: { get: () => Object.keys(record).length, enumerable: false },
|
|
});
|
|
return Object.freeze(record);
|
|
}
|
|
|
|
export const TAXONOMY_GROUPS = attachLookupHelpers(${JSON.stringify(groupsMap, null, 2)});
|
|
|
|
export const TAXONOMY_SUBJECTS = attachLookupHelpers(${JSON.stringify(subjectsMap, null, 2)});
|
|
|
|
export const CHAPTER_TAXONOMY_INDEX = attachLookupHelpers(${JSON.stringify(chapterTaxonomyIndex, null, 2)});
|
|
|
|
/**
|
|
* Resolves any chapter URL, pathname, or slug (including \`/vol1/01_introduction/\`,
|
|
* \`/vol1/chapters/01_introduction/01_introduction.html\`, or \`vol1/01_introduction\`)
|
|
* to its canonical chapter taxonomy entry with subject details.
|
|
*/
|
|
export function resolveChapterTaxonomy(urlOrPath = '') {
|
|
if (typeof urlOrPath !== 'string' || !urlOrPath.trim()) return null;
|
|
const raw = urlOrPath.trim();
|
|
|
|
// Direct slug lookup first
|
|
if (CHAPTER_TAXONOMY_INDEX[raw]) {
|
|
return enrichChapterEntry(CHAPTER_TAXONOMY_INDEX[raw]);
|
|
}
|
|
|
|
let pathname = raw;
|
|
try {
|
|
if (/^https?:\\/\\//i.test(raw)) {
|
|
pathname = new URL(raw).pathname;
|
|
}
|
|
} catch {
|
|
pathname = raw;
|
|
}
|
|
|
|
const normalized = pathname.replace(/\\\\/g, '/');
|
|
|
|
// Match /volN/chapters/NN_slug or /volN/NN_slug
|
|
const match = normalized.match(/(?:^|\\b|\\/)(vol[1-4])\\/(?:chapters\\/)?([0-9]{2}_[a-z0-9_]+)(?:\\/|\\.html|$|#|\\?)/i);
|
|
if (match) {
|
|
const slug = \`\${match[1].toLowerCase()}/\${match[2].toLowerCase()}\`;
|
|
if (CHAPTER_TAXONOMY_INDEX[slug]) {
|
|
return enrichChapterEntry(CHAPTER_TAXONOMY_INDEX[slug]);
|
|
}
|
|
}
|
|
|
|
// Fallback: check if any indexed chapter slug or unique chapter directory appears in the path
|
|
for (const [slug, entry] of Object.entries(CHAPTER_TAXONOMY_INDEX)) {
|
|
const [, folder] = slug.split('/');
|
|
if (normalized.includes(slug)) {
|
|
return enrichChapterEntry(entry);
|
|
}
|
|
if (folder && normalized.includes(\`/\${entry.volume}/\`) && normalized.includes(folder)) {
|
|
return enrichChapterEntry(entry);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function enrichChapterEntry(entry) {
|
|
if (!entry) return null;
|
|
const primaryGroupInfo = TAXONOMY_GROUPS[entry.primaryGroup] || null;
|
|
const subjectDetails = (entry.subjects || [])
|
|
.map((code) => TAXONOMY_SUBJECTS[code])
|
|
.filter(Boolean);
|
|
return {
|
|
...entry,
|
|
groupTitle: primaryGroupInfo?.name || entry.primaryGroup,
|
|
primaryGroupInfo,
|
|
subjectDetails,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Returns a concise (~180-240 char) grounding line when a chapter matches,
|
|
* or a general 1-line 5-Group overview when on a non-chapter page.
|
|
*/
|
|
export function formatChapterTaxonomyPrompt(urlOrPath = '') {
|
|
const chapter = resolveChapterTaxonomy(urlOrPath);
|
|
if (chapter) {
|
|
const shortCode = chapter.slug.replace(/^((?:vol[1-4])\\/[0-9]{2})_.*$/, '$1');
|
|
const topSubjects = chapter.subjectDetails
|
|
.slice(0, 3)
|
|
.map((s) => \`\${s.code} \${s.name}\`)
|
|
.join(', ');
|
|
return \`Active chapter ML Systems Taxonomy (\${shortCode} \${chapter.title}): Groups \${chapter.groups.join(', ')} | Subjects \${topSubjects}. Situate explanations and trade-offs across Foundations (F), Stack (S), Lifecycle (P), Qualities (Q), and System Types (T).\`;
|
|
}
|
|
return 'ML Systems Taxonomy lens: situate mechanisms and trade-offs across the 5 Groups — Foundations (F), Stack (S), Lifecycle (P), Qualities (Q), and System Types (T).';
|
|
}
|
|
|
|
/**
|
|
* Returns matching taxonomy items (starting with a '🧭 Current chapter taxonomy overview'
|
|
* item and the active chapter's subjects, followed by matching subjects from TAXONOMY_SUBJECTS)
|
|
* for use in + Context and @ mentions.
|
|
*/
|
|
export function searchTaxonomyItems(query = '', currentUrl = '') {
|
|
const normalizedQuery = String(query || '').trim().toLowerCase();
|
|
const chapter = resolveChapterTaxonomy(currentUrl);
|
|
const results = [];
|
|
const seenKeys = new Set();
|
|
|
|
const overviewTitle = 'Current chapter taxonomy overview';
|
|
const overviewContent = chapter
|
|
? \`\${formatChapterTaxonomyPrompt(currentUrl)}\\n\\nPrimary Group: \${chapter.primaryGroup} (\${TAXONOMY_GROUPS[chapter.primaryGroup]?.name || chapter.primaryGroup})\\nSubjects: \${chapter.subjectDetails.map((s) => \`\${s.code} \${s.name}\`).join(', ')}\`
|
|
: 'ML Systems Taxonomy Overview: Foundations (F: workloads, requirements, models, performance models), Stack (S: hardware, compilers, parallelism, fleet), Lifecycle (P: data, training, evaluation, serving, MLOps), Qualities (Q: performance, reliability, security, safety, responsible AI, sustainability), System Types (T: agentic and embodied/physical systems).';
|
|
|
|
const matchesOverview = !normalizedQuery
|
|
|| 'taxonomy'.includes(normalizedQuery)
|
|
|| overviewTitle.toLowerCase().includes(normalizedQuery)
|
|
|| (chapter && (
|
|
chapter.slug.toLowerCase().includes(normalizedQuery)
|
|
|| chapter.title.toLowerCase().includes(normalizedQuery)
|
|
));
|
|
|
|
if (matchesOverview) {
|
|
results.push({
|
|
type: 'taxonomy',
|
|
icon: '🧭',
|
|
code: 'overview',
|
|
key: overviewTitle,
|
|
title: overviewTitle,
|
|
content: overviewContent,
|
|
value: {
|
|
title: overviewTitle,
|
|
content: overviewContent,
|
|
text: overviewContent,
|
|
},
|
|
});
|
|
seenKeys.add(overviewTitle);
|
|
}
|
|
|
|
function makeSubjectItem(subject) {
|
|
const title = \`\${subject.code} \${subject.name}\`;
|
|
const groupInfo = TAXONOMY_GROUPS[subject.group];
|
|
const content = \`\${subject.code} \${subject.name} [Group \${subject.group} \${groupInfo ? groupInfo.name : ''} · \${subject.areaCode} \${subject.areaName}]: \${subject.summary}\`;
|
|
return {
|
|
type: 'taxonomy',
|
|
icon: '🧭',
|
|
code: subject.code,
|
|
group: subject.group,
|
|
key: title,
|
|
title,
|
|
content,
|
|
value: {
|
|
title,
|
|
content,
|
|
text: content,
|
|
},
|
|
};
|
|
}
|
|
|
|
function subjectMatches(subject) {
|
|
if (!normalizedQuery || 'taxonomy'.includes(normalizedQuery)) return true;
|
|
const haystack = \`\${subject.code} \${subject.name} \${subject.areaCode} \${subject.areaName} \${subject.summary} \${TAXONOMY_GROUPS[subject.group]?.name || ''}\`.toLowerCase();
|
|
return haystack.includes(normalizedQuery);
|
|
}
|
|
|
|
if (chapter) {
|
|
for (const subject of chapter.subjectDetails) {
|
|
const item = makeSubjectItem(subject);
|
|
if (!seenKeys.has(item.key) && subjectMatches(subject)) {
|
|
results.push(item);
|
|
seenKeys.add(item.key);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const subject of Object.values(TAXONOMY_SUBJECTS)) {
|
|
const item = makeSubjectItem(subject);
|
|
if (!seenKeys.has(item.key) && subjectMatches(subject)) {
|
|
results.push(item);
|
|
seenKeys.add(item.key);
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
`;
|
|
|
|
await mkdir(dirname(OUTPUT_PATH), { recursive: true });
|
|
await writeFile(OUTPUT_PATH, moduleSource, 'utf8');
|
|
console.log(
|
|
`Generated ${OUTPUT_PATH} (${Object.keys(groupsMap).length} groups, ${Object.keys(subjectsMap).length} subjects, ${Object.keys(chapterTaxonomyIndex).length} chapters)`
|
|
);
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|