Skip to content

Commit 58f3959

Browse files
feat: advanced mdx support (#629)
* feat: advanced mdx * chore: changeset
1 parent e926df1 commit 58f3959

7 files changed

Lines changed: 1121 additions & 12 deletions

File tree

.changeset/tasty-swans-explode.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@lingo.dev/_spec": minor
3+
"lingo.dev": minor
4+
---
5+
6+
advanced mdx support (shout out to @ZYJLiu!)

packages/cli/i18n.json

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,9 @@
11
{
2-
"version": 1.5,
3-
"provider": {
4-
"id": "anthropic",
5-
"model": "claude-3-7-sonnet-latest",
6-
"prompt": "You're translating text from {source} to {target}."
7-
},
2+
"version": 1.6,
83
"locale": {
94
"source": "en",
10-
"targets": ["es"]
11-
},
12-
"buckets": {
13-
"json": {
14-
"include": ["demo/json/[locale].json"]
15-
}
5+
"targets": ["de"]
166
},
7+
"buckets": {},
178
"$schema": "https://lingo.dev/schema/i18n.json"
189
}

packages/cli/package.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
"@lingo.dev/_spec": "workspace:*",
6767
"@modelcontextprotocol/sdk": "^1.5.0",
6868
"@paralleldrive/cuid2": "^2.2.2",
69+
"@types/mdast": "^4.0.4",
6970
"ai": "^4.3.2",
7071
"bitbucket": "^2.12.0",
7172
"chalk": "^5.4.1",
@@ -95,6 +96,11 @@
9596
"markdown-it": "^14.1.0",
9697
"markdown-it-front-matter": "^0.2.4",
9798
"marked": "^15.0.6",
99+
"mdast-util-from-markdown": "^2.0.2",
100+
"mdast-util-frontmatter": "^2.0.1",
101+
"mdast-util-gfm": "^3.1.0",
102+
"mdast-util-mdx": "^3.0.0",
103+
"mdast-util-to-markdown": "^2.1.2",
98104
"node-webvtt": "^1.9.4",
99105
"object-hash": "^3.0.0",
100106
"octokit": "^4.0.2",
@@ -106,9 +112,18 @@
106112
"posthog-node": "^4.11.2",
107113
"prettier": "^3.4.2",
108114
"properties-parser": "^0.6.0",
115+
"remark-frontmatter": "^5.0.0",
116+
"remark-gfm": "^4.0.1",
117+
"remark-mdx": "^3.1.0",
118+
"remark-mdx-frontmatter": "^5.1.0",
119+
"remark-parse": "^11.0.0",
120+
"remark-stringify": "^11.0.0",
109121
"slugify": "^1.6.6",
110122
"srt-parser-2": "^1.2.3",
111123
"typescript": "^5.7.2",
124+
"unified": "^11.0.5",
125+
"unist-util-visit": "^5.0.0",
126+
"vfile": "^6.0.3",
112127
"vitest": "^2.1.8",
113128
"xliff": "^6.2.1",
114129
"xml2js": "^0.6.2",

packages/cli/src/cli/loaders/index.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import createAndroidLoader from "./android";
1212
import createCsvLoader from "./csv";
1313
import createHtmlLoader from "./html";
1414
import createMarkdownLoader from "./markdown";
15+
import { createMdxFormatLoader, createMdxStructureLoader } from "./mdx";
1516
import createPropertiesLoader from "./properties";
1617
import createXcodeStringsLoader from "./xcode-strings";
1718
import createXcodeStringsdictLoader from "./xcode-stringsdict";
@@ -91,6 +92,16 @@ export default function createBucketLoader(
9192
createSyncLoader(),
9293
createUnlocalizableLoader(options.isCacheRestore, options.returnUnlocalizedKeys),
9394
);
95+
case "mdx":
96+
return composeLoaders(
97+
createTextFileLoader(bucketPathPattern),
98+
createPrettierLoader({ parser: "mdx", bucketPathPattern }),
99+
createMdxFormatLoader(),
100+
createFlatLoader(),
101+
createMdxStructureLoader(),
102+
createSyncLoader(),
103+
createUnlocalizableLoader(options.isCacheRestore, options.returnUnlocalizedKeys),
104+
);
94105
case "po":
95106
return composeLoaders(
96107
createTextFileLoader(bucketPathPattern),
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import _ from "lodash";
2+
import { unified } from "unified";
3+
import remarkParse from "remark-parse";
4+
import remarkMdx from "remark-mdx";
5+
import remarkFrontmatter from "remark-frontmatter";
6+
import remarkGfm from "remark-gfm";
7+
import remarkStringify from "remark-stringify";
8+
import remarkMdxFrontmatter from "remark-mdx-frontmatter";
9+
import { VFile } from "vfile";
10+
import { Root } from "mdast";
11+
import { ILoader } from "./_types";
12+
import { composeLoaders, createLoader } from "./_utils";
13+
14+
// Define MDX node types
15+
type MdxNode = {
16+
type: string;
17+
[key: string]: any;
18+
};
19+
20+
type MdxTree = {
21+
type: "root";
22+
children: MdxNode[];
23+
};
24+
25+
export function createMdxFormatLoader(): ILoader<string, Record<string, any>> {
26+
// Create a unified processor for MDX parsing with all required plugins
27+
const parser = unified()
28+
.use(remarkParse)
29+
.use(remarkMdx)
30+
.use(remarkFrontmatter, ["yaml"])
31+
.use(remarkMdxFrontmatter)
32+
.use(remarkGfm);
33+
34+
// Create a unified processor for MDX serialization
35+
const serializer = unified()
36+
.use(remarkStringify)
37+
.use(remarkMdx)
38+
.use(remarkFrontmatter, ["yaml"])
39+
.use(remarkMdxFrontmatter)
40+
.use(remarkGfm);
41+
42+
return createLoader({
43+
async pull(locale, input) {
44+
// Parse the MDX content into an AST
45+
const file = new VFile(input);
46+
const ast = parser.parse(file);
47+
48+
// Instead of returning the AST directly, convert it to a plain object
49+
// This ensures compatibility with the ILoader interface
50+
return JSON.parse(JSON.stringify(ast));
51+
},
52+
53+
async push(locale, data) {
54+
// Recreate an AST from the plain object
55+
const ast = data as unknown as Root;
56+
57+
// Serialize the AST back to MDX content
58+
const file = serializer.stringify(ast);
59+
return String(file);
60+
},
61+
});
62+
}
63+
64+
export function createMdxStructureLoader(): ILoader<Record<string, any>, Record<string, string>> {
65+
return createLoader({
66+
async pull(locale, input) {
67+
const result = _.chain(input)
68+
.pickBy((value, key) => key.endsWith("/value"))
69+
.value();
70+
71+
return result;
72+
},
73+
async push(locale, data, originalInput) {
74+
const result = _.merge({}, originalInput, data);
75+
76+
return result;
77+
},
78+
});
79+
}

packages/spec/src/formats.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ export const bucketTypes = [
77
"html",
88
"json",
99
"markdown",
10+
"mdx",
1011
"xcode-strings",
1112
"xcode-stringsdict",
1213
"xcode-xcstrings",

0 commit comments

Comments
 (0)