refactor(core): load generators by import specifier

Author: avivkeller

.changeset/specifier-generator-loading.md

@@ -0,0 +1,11 @@
+---
+'@node-core/doc-kit': minor
+---
+
+Generators are now loaded dynamically by import specifier instead of a static
+registry. `--target` accepts either a built-in shorthand name (`web`,
+`legacy-html`, …) or any import specifier resolving to a generator module
+(e.g. `some-package/generator` or `./my-generator.mjs`), and a generator's
+`dependsOn` is now a full import specifier. This lays the groundwork for
+splitting the built-in generators into separate packages and enables
+third-party generator packages.

docs/configuration.md

@@ -36,7 +36,9 @@ uses the `doc-kit` property:
 
 ```javascript
 export default {
-  // targets, alternatively supplied by command line flags
+  // Targets, alternatively supplied by command line flags. Each entry is
+  // either a built-in shorthand name or an import specifier resolving to a
+  // generator module (e.g. '@my-scope/my-package/my-generator').
   target: ['orama-db', 'web'],
   global: {
     version: '20.0.0',

docs/generators.md

@@ -69,24 +69,26 @@ export type Generator = GeneratorMetadata<
 
 ### Step 3: Define Generator Metadata
 
-Create the generator metadata in `index.mjs` using `createLazyGenerator`:
+A generator module's default export is a plain object with its metadata and
+implementation. Create it in `index.mjs`:
 
 ```javascript
 // packages/core/src/generators/my-format/index.mjs
-import { createLazyGenerator } from '../../utils/generators.mjs';
+import { generate } from './generate.mjs';
 
 /**
  * Generates output in MyFormat.
  *
  * @type {import('./types').Generator}
  */
-export default createLazyGenerator({
+export default {
   name: 'my-format',
 
   description: 'Generates documentation in MyFormat',
 
-  // This generator depends on the metadata generator
-  dependsOn: 'metadata',
+  // This generator depends on the metadata generator. Dependencies are
+  // declared as import specifiers, so they can live in any package.
+  dependsOn: '@node-core/doc-kit/metadata',
 
   defaultConfiguration: {
     // If your generator supports a custom configuration, define the defaults here
@@ -96,7 +98,9 @@ export default createLazyGenerator({
     // To override the defaults, they can be specified here
     ref: 'overriddenRef',
   },
-});
+
+  generate,
+};
 ```
 
 ### Step 4: Implement the Generator Logic
@@ -147,28 +151,34 @@ function transformToMyFormat(entries, version) {
 }
 ```
 
-### Step 5: Register the Generator
+### Step 5: Make the Generator Loadable
 
-Add your generator to the exports in `packages/core/src/generators/index.mjs`:
+Generators are loaded dynamically by import specifier. Anything that resolves
+to a module whose default export is a generator works as a `--target`:
 
-```javascript
-// For public generators (available via CLI)
-import myFormat from './my-format/index.mjs';
+```bash
+# A package (subpath) export
+doc-kit generate -t @my-scope/my-package/my-format ...
+
+# A local file
+doc-kit generate -t ./generators/my-format/index.mjs ...
+```
+
+Built-in generators additionally get a shorthand alias in
+`packages/core/src/generators/index.mjs`, which maps the name users type to
+the import specifier it resolves to:
 
+```javascript
 export const publicGenerators = {
-  'json-simple': jsonSimple,
-  'my-format': myFormat, // Add this
+  'json-simple': '@node-core/doc-kit/json-simple',
+  'my-format': '@node-core/doc-kit/my-format', // Add this
   // ... other generators
 };
-
-// For internal generators (used only as dependencies)
-const internalGenerators = {
-  ast,
-  metadata,
-  // ... internal generators
-};
 ```
 
+If the generator lives in this repository, also add a matching subpath to the
+`exports` map of its package's `package.json`.
+
 ## Parallel Processing with Workers
 
 For generators processing large datasets, implement parallel processing using worker threads.
@@ -179,21 +189,24 @@ First, define the generator metadata in `index.mjs`:
 
 ```javascript
 // packages/core/src/generators/parallel-generator/index.mjs
-import { createLazyGenerator } from '../../utils/generators.mjs';
+import { generate, processChunk } from './generate.mjs';
 
 /**
  * @type {import('./types').Generator}
  */
-export default createLazyGenerator({
+export default {
   name: 'parallel-generator',
 
   description: 'Processes data in parallel',
 
-  dependsOn: 'metadata',
+  dependsOn: '@node-core/doc-kit/metadata',
 
   // Indicates this generator has a processChunk implementation
   hasParallelProcessor: true,
-});
+
+  generate,
+  processChunk,
+};
 ```
 
 Then, implement both `processChunk` and `generate` in `generate.mjs`:
@@ -273,20 +286,23 @@ Define the generator metadata in `index.mjs`:
 
 ```javascript
 // packages/core/src/generators/streaming-generator/index.mjs
-import { createLazyGenerator } from '../../utils/generators.mjs';
+import { generate, processChunk } from './generate.mjs';
 
 /**
  * @type {import('./types').Generator}
  */
-export default createLazyGenerator({
+export default {
   name: 'streaming-generator',
 
   description: 'Streams results as they are ready',
 
-  dependsOn: 'metadata',
+  dependsOn: '@node-core/doc-kit/metadata',
 
   hasParallelProcessor: true,
-});
+
+  generate,
+  processChunk,
+};
 ```
 
 Implement the generator in `generate.mjs`:
@@ -331,18 +347,20 @@ Generator metadata in `index.mjs`:
 
 ```javascript
 // packages/core/src/generators/batch-generator/index.mjs
-import { createLazyGenerator } from '../../utils/generators.mjs';
+import { generate } from './generate.mjs';
 
 /**
  * @type {import('./types').Generator}
  */
-export default createLazyGenerator({
+export default {
   name: 'batch-generator',
 
   description: 'Requires all input at once',
 
-  dependsOn: 'jsx-ast',
-});
+  dependsOn: '@node-core/doc-kit/jsx-ast',
+
+  generate,
+};
 ```
 
 Implementation in `generate.mjs`:
@@ -378,15 +396,19 @@ Use non-streaming when:
 In `index.mjs`:
 
 ```javascript
-import { createLazyGenerator } from '../../utils/generators.mjs';
+import { generate } from './generate.mjs';
 
-export default createLazyGenerator({
+export default {
   name: 'my-generator',
 
-  dependsOn: 'metadata', // This generator requires metadata output
+  // This generator requires the metadata generator's output. The dependency
+  // is an import specifier, so it may point at any installed package.
+  dependsOn: '@node-core/doc-kit/metadata',
 
   // ... other metadata
-});
+
+  generate,
+};
 ```
 
 In `generate.mjs`:
@@ -402,27 +424,27 @@ export async function generate(input, worker) {
 ```javascript
 // Step 1: Parse markdown to AST
 // packages/core/src/generators/ast/index.mjs
-export default createLazyGenerator({
+export default {
   name: 'ast',
-  dependsOn: undefined,  // No dependency
+  dependsOn: undefined, // No dependency
   // Processes raw markdown files
-});
+};
 
 // Step 2: Extract metadata from AST
 // packages/core/src/generators/metadata/index.mjs
-export default createLazyGenerator({
+export default {
   name: 'metadata',
-  dependsOn: 'ast',  // Depends on AST
+  dependsOn: '@node-core/doc-kit/ast', // Depends on AST
   // Processes AST output
-});
+};
 
 // Step 3: Generate HTML from metadata
 // packages/core/src/generators/html-generator/index.mjs
-export default createLazyGenerator({
+export default {
   name: 'html-generator',
-  dependsOn: 'metadata',  // Depends on metadata
+  dependsOn: '@node-core/doc-kit/metadata', // Depends on metadata
   // Processes metadata output
-});
+};
 ```
 
 ### Multiple Consumers

packages/core/bin/commands/generate.mjs

@@ -36,8 +36,11 @@ export default new Command('generate')
     new Option('-i, --input <patterns...>', 'Input file patterns (glob)')
   )
   .addOption(
-    new Option('-t, --target <generator...>', 'Target generator(s)').choices(
-      Object.keys(publicGenerators)
+    new Option(
+      '-t, --target <generator...>',
+      'Target generator(s): a built-in name ' +
+        `(${Object.keys(publicGenerators).join(', ')}) ` +
+        'or an import specifier for a custom generator'
     )
   )
   .addOption(

packages/core/package.json

@@ -16,6 +16,29 @@
     "watch": "node --watch bin/cli.mjs"
   },
   "main": "./src/generators.mjs",
+  "exports": {
+    ".": "./src/generators.mjs",
+    "./addon-verify": "./src/generators/addon-verify/index.mjs",
+    "./api-links": "./src/generators/api-links/index.mjs",
+    "./ast": "./src/generators/ast/index.mjs",
+    "./ast-js": "./src/generators/ast-js/index.mjs",
+    "./json-simple": "./src/generators/json-simple/index.mjs",
+    "./jsx-ast": "./src/generators/jsx-ast/index.mjs",
+    "./legacy-html": "./src/generators/legacy-html/index.mjs",
+    "./legacy-html-all": "./src/generators/legacy-html-all/index.mjs",
+    "./legacy-json": "./src/generators/legacy-json/index.mjs",
+    "./legacy-json-all": "./src/generators/legacy-json-all/index.mjs",
+    "./llms-txt": "./src/generators/llms-txt/index.mjs",
+    "./man-page": "./src/generators/man-page/index.mjs",
+    "./metadata": "./src/generators/metadata/index.mjs",
+    "./orama-db": "./src/generators/orama-db/index.mjs",
+    "./sitemap": "./src/generators/sitemap/index.mjs",
+    "./web": "./src/generators/web/index.mjs",
+    "./package.json": "./package.json",
+    "./shiki.config.mjs": "./shiki.config.mjs",
+    "./src/*": "./src/*",
+    "./*": "./src/*"
+  },
   "bin": {
     "doc-kit": "./bin/cli.mjs"
   },