Skip to content

Commit cac5429

Browse files
authored
feat(cli): progressively push translated chunks to locale files (#955)
* feat(wip): progressive push as chunks are processed * chore(cli): remove "isCacheRestore" option * fix(cli): only re-add unlocalized keys on push * fix(cli): only re-add locked keys on push * fix(cli): only re-add locale keys on push * fix(cli): only re-add metadata keys on push * fix(cli): only re-add removed keys (without "/value") on push * chore(cli): test xcstrings loader * fix(cli): ensure key order via distinct loader * fix(cli): refactor run logic for distinct and unified locale files * feat(cli): purge command Removes translations for given --bucket, --file, --key, --locale Asks for interactive confirmation for each file unless --yes-really is present. * Apply suggestions from code review * chore: add changeset * fix(cli): purge for xcode-xcstrings * chore: format * fix: remove isCacheRestore from EJS loader
1 parent 0fc6385 commit cac5429

28 files changed

Lines changed: 1440 additions & 423 deletions

.changeset/slow-mirrors-exist.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"lingo.dev": patch
3+
---
4+
5+
progressive push as chunks are processed

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"@types/babel__traverse": "^7.20.7",
2121
"commitlint": "^19.7.1",
2222
"husky": "^9.1.7",
23+
"prettier": "^3.4.2",
2324
"turbo": "^2.5.0"
2425
},
2526
"dependencies": {

packages/cli/src/cli/cmd/cleanup.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ export default new Command()
6060
bucket.type,
6161
bucketConfig.pathPattern,
6262
{
63-
isCacheRestore: false,
6463
defaultLocale: sourceLocale,
6564
},
6665
);

packages/cli/src/cli/cmd/i18n.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ export default new Command()
176176
bucket.type,
177177
bucketPath.pathPattern,
178178
{
179-
isCacheRestore: false,
180179
defaultLocale: sourceLocale,
181180
injectLocale: bucket.injectLocale,
182181
},
@@ -215,7 +214,6 @@ export default new Command()
215214
bucket.type,
216215
bucketPath.pathPattern,
217216
{
218-
isCacheRestore: false,
219217
defaultLocale: sourceLocale,
220218
returnUnlocalizedKeys: true,
221219
injectLocale: bucket.injectLocale,
@@ -324,7 +322,6 @@ export default new Command()
324322
bucket.type,
325323
bucketPath.pathPattern,
326324
{
327-
isCacheRestore: false,
328325
defaultLocale: sourceLocale,
329326
injectLocale: bucket.injectLocale,
330327
},

packages/cli/src/cli/cmd/lockfile.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ export default new Command()
3535
bucket.type,
3636
bucketConfig.pathPattern,
3737
{
38-
isCacheRestore: false,
3938
defaultLocale: sourceLocale,
4039
},
4140
);

packages/cli/src/cli/cmd/purge.ts

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
import { Command } from "interactive-commander";
2+
import _ from "lodash";
3+
import Ora from "ora";
4+
import { getConfig } from "../utils/config";
5+
import { getBuckets } from "../utils/buckets";
6+
import { resolveOverriddenLocale } from "@lingo.dev/_spec";
7+
import createBucketLoader from "../loaders";
8+
import { minimatch } from "minimatch";
9+
import { confirm } from "@inquirer/prompts";
10+
11+
interface PurgeOptions {
12+
bucket?: string[];
13+
file?: string[];
14+
key?: string;
15+
locale?: string[];
16+
yesReally?: boolean;
17+
}
18+
19+
export default new Command()
20+
.command("purge")
21+
.description(
22+
"Remove translations for given --bucket, --file, --key, --locale",
23+
)
24+
.helpOption("-h, --help", "Show help")
25+
.option(
26+
"--bucket <bucket>",
27+
"Bucket to process",
28+
(val: string, prev: string[]) => (prev ? [...prev, val] : [val]),
29+
)
30+
.option(
31+
"--file [files...]",
32+
"File(s) to process. Only process files that match the given glob pattern(s).",
33+
)
34+
.option(
35+
"--key <key>",
36+
"Key to remove. Remove all translation keys matching the given glob pattern.",
37+
)
38+
.option(
39+
"--locale <locale>",
40+
"Locale to process",
41+
(val: string, prev: string[]) => (prev ? [...prev, val] : [val]),
42+
)
43+
.option(
44+
"--yes-really",
45+
"Skip interactive confirmation and delete without asking.",
46+
)
47+
.action(async function (options: PurgeOptions) {
48+
const ora = Ora();
49+
try {
50+
ora.start("Loading configuration...");
51+
const i18nConfig = getConfig();
52+
if (!i18nConfig) {
53+
throw new Error("i18n.json not found. Please run `lingo.dev init`.");
54+
}
55+
ora.succeed("Configuration loaded");
56+
57+
let buckets = getBuckets(i18nConfig);
58+
if (options.bucket && options.bucket.length) {
59+
buckets = buckets.filter((bucket) =>
60+
options.bucket!.includes(bucket.type),
61+
);
62+
}
63+
if (options.file && options.file.length) {
64+
buckets = buckets
65+
.map((bucket) => {
66+
const paths = bucket.paths.filter((bucketPath) =>
67+
options.file?.some((f) => bucketPath.pathPattern.includes(f)),
68+
);
69+
return { ...bucket, paths };
70+
})
71+
.filter((bucket) => bucket.paths.length > 0);
72+
if (buckets.length === 0) {
73+
ora.fail("All files were filtered out by --file option.");
74+
process.exit(1);
75+
}
76+
}
77+
const sourceLocale = i18nConfig.locale.source;
78+
const targetLocales =
79+
options.locale && options.locale.length
80+
? options.locale
81+
: i18nConfig.locale.targets;
82+
let removedAny = false;
83+
for (const bucket of buckets) {
84+
console.log();
85+
ora.info(`Processing bucket: ${bucket.type}`);
86+
for (const bucketPath of bucket.paths) {
87+
for (const _targetLocale of targetLocales) {
88+
const targetLocale = resolveOverriddenLocale(
89+
_targetLocale,
90+
bucketPath.delimiter,
91+
);
92+
const bucketOra = Ora({ indent: 2 }).start(
93+
`Processing path: ${bucketPath.pathPattern} [${targetLocale}]`,
94+
);
95+
try {
96+
const bucketLoader = createBucketLoader(
97+
bucket.type,
98+
bucketPath.pathPattern,
99+
{
100+
defaultLocale: sourceLocale,
101+
injectLocale: bucket.injectLocale,
102+
},
103+
bucket.lockedKeys,
104+
bucket.lockedPatterns,
105+
bucket.ignoredKeys,
106+
);
107+
await bucketLoader.init();
108+
bucketLoader.setDefaultLocale(sourceLocale);
109+
await bucketLoader.pull(sourceLocale);
110+
let targetData = await bucketLoader.pull(targetLocale);
111+
if (!targetData || Object.keys(targetData).length === 0) {
112+
bucketOra.info(
113+
`No translations found for ${bucketPath.pathPattern} [${targetLocale}]`,
114+
);
115+
continue;
116+
}
117+
let newData = { ...targetData };
118+
let keysToRemove: string[] = [];
119+
if (options.key) {
120+
// minimatch for key patterns
121+
keysToRemove = Object.keys(newData).filter((k) =>
122+
minimatch(k, options.key!),
123+
);
124+
} else {
125+
// No key specified: remove all keys
126+
keysToRemove = Object.keys(newData);
127+
}
128+
if (keysToRemove.length > 0) {
129+
// Show what will be deleted
130+
if (options.key) {
131+
bucketOra.info(
132+
`About to delete ${keysToRemove.length} key(s) matching '${options.key}' from ${bucketPath.pathPattern} [${targetLocale}]:\n ${keysToRemove.slice(0, 10).join(", ")}${keysToRemove.length > 10 ? ", ..." : ""}`,
133+
);
134+
} else {
135+
bucketOra.info(
136+
`About to delete all (${keysToRemove.length}) keys from ${bucketPath.pathPattern} [${targetLocale}]`,
137+
);
138+
}
139+
140+
if (!options.yesReally) {
141+
bucketOra.warn(
142+
"This is a destructive operation. If you are sure, type 'y' to continue. (Use --yes-really to skip this check.)",
143+
);
144+
const confirmed = await confirm({
145+
message: `Delete these keys from ${bucketPath.pathPattern} [${targetLocale}]?`,
146+
default: false,
147+
});
148+
if (!confirmed) {
149+
bucketOra.info("Skipped by user.");
150+
continue;
151+
}
152+
}
153+
for (const key of keysToRemove) {
154+
delete newData[key];
155+
}
156+
removedAny = true;
157+
await bucketLoader.push(targetLocale, newData);
158+
if (options.key) {
159+
bucketOra.succeed(
160+
`Removed ${keysToRemove.length} key(s) matching '${options.key}' from ${bucketPath.pathPattern} [${targetLocale}]`,
161+
);
162+
} else {
163+
bucketOra.succeed(
164+
`Removed all keys (${keysToRemove.length}) from ${bucketPath.pathPattern} [${targetLocale}]`,
165+
);
166+
}
167+
} else if (options.key) {
168+
bucketOra.info(
169+
`No keys matching '${options.key}' found in ${bucketPath.pathPattern} [${targetLocale}]`,
170+
);
171+
} else {
172+
bucketOra.info("No keys to remove.");
173+
}
174+
} catch (error) {
175+
const err = error as Error;
176+
bucketOra.fail(`Failed: ${err.message}`);
177+
}
178+
}
179+
}
180+
}
181+
if (!removedAny) {
182+
ora.info("No keys were removed.");
183+
} else {
184+
ora.succeed("Purge completed.");
185+
}
186+
} catch (error) {
187+
const err = error as Error;
188+
ora.fail(err.message);
189+
process.exit(1);
190+
}
191+
});

packages/cli/src/cli/cmd/run/execute.ts

Lines changed: 50 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { colors } from "../../constants";
88
import { CmdRunContext, CmdRunTask, CmdRunTaskResult } from "./_types";
99
import { commonTaskRendererOptions } from "./_const";
1010
import createBucketLoader from "../../loaders";
11-
import { createDeltaProcessor } from "../../utils/delta";
11+
import { createDeltaProcessor, Delta } from "../../utils/delta";
1212

1313
const MAX_WORKER_COUNT = 10;
1414

@@ -123,7 +123,6 @@ function createLoaderForTask(assignedTask: CmdRunTask) {
123123
assignedTask.bucketPathPattern,
124124
{
125125
defaultLocale: assignedTask.sourceLocale,
126-
isCacheRestore: false,
127126
injectLocale: assignedTask.injectLocale,
128127
},
129128
assignedTask.lockedKeys,
@@ -199,40 +198,58 @@ function createWorkerTask(args: {
199198
targetData,
200199
processableData,
201200
},
202-
(progress) => {
201+
async (progress, _sourceChunk, processedChunk) => {
202+
// write translated chunks as they are received from LLM
203+
await args.ioLimiter(async () => {
204+
// pull the latest source data before pushing for buckets that store all locales in a single file
205+
await bucketLoader.pull(assignedTask.sourceLocale);
206+
// pull the latest target data to include all already processed chunks
207+
const latestTargetData = await bucketLoader.pull(
208+
assignedTask.targetLocale,
209+
);
210+
// add the new chunk to target data
211+
const _partialData = _.merge(
212+
{},
213+
latestTargetData,
214+
processedChunk,
215+
);
216+
// process renamed keys
217+
const finalChunkTargetData = processRenamedKeys(
218+
delta,
219+
_partialData,
220+
);
221+
// push final chunk to the target locale
222+
await bucketLoader.push(
223+
assignedTask.targetLocale,
224+
finalChunkTargetData,
225+
);
226+
});
227+
203228
subTask.title = createWorkerStatusMessage({
204229
assignedTask,
205230
percentage: progress,
206231
});
207232
},
208233
);
209234

210-
let finalTargetData = _.merge(
235+
const finalTargetData = _.merge(
211236
{},
212237
sourceData,
213238
targetData,
214239
processedTargetData,
215240
);
216-
217-
finalTargetData = _.chain(finalTargetData)
218-
.entries()
219-
.map(([key, value]) => {
220-
const renaming = delta.renamed.find(
221-
([oldKey]) => oldKey === key,
222-
);
223-
if (!renaming) {
224-
return [key, value];
225-
}
226-
return [renaming[1], value];
227-
})
228-
.fromPairs()
229-
.value();
241+
const finalRenamedTargetData = processRenamedKeys(
242+
delta,
243+
finalTargetData,
244+
);
230245

231246
await args.ioLimiter(async () => {
247+
// not all localizers have progress callback (eg. explicit localizer),
248+
// the final target data might not be pushed yet - push now to ensure it's up to date
232249
await bucketLoader.pull(assignedTask.sourceLocale);
233250
await bucketLoader.push(
234251
assignedTask.targetLocale,
235-
finalTargetData,
252+
finalRenamedTargetData,
236253
);
237254

238255
const checksums =
@@ -265,3 +282,17 @@ function countTasks(
265282
predicate(task, result),
266283
).length;
267284
}
285+
286+
function processRenamedKeys(delta: Delta, targetData: Record<string, string>) {
287+
return _.chain(targetData)
288+
.entries()
289+
.map(([key, value]) => {
290+
const renaming = delta.renamed.find(([oldKey]) => oldKey === key);
291+
if (!renaming) {
292+
return [key, value];
293+
}
294+
return [renaming[1], value];
295+
})
296+
.fromPairs()
297+
.value();
298+
}

packages/cli/src/cli/cmd/status.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,6 @@ export default new Command()
185185
bucket.type,
186186
bucketPath.pathPattern,
187187
{
188-
isCacheRestore: false,
189188
defaultLocale: sourceLocale,
190189
injectLocale: bucket.injectLocale,
191190
},
@@ -351,6 +350,9 @@ export default new Command()
351350

352351
if (flags.verbose) {
353352
if (missingKeys.length > 0) {
353+
console.log(
354+
` ${chalk.red(`Missing:`)} ${missingKeys.length} keys, ~${wordsToTranslate} words`,
355+
);
354356
console.log(
355357
` ${chalk.red(`Missing:`)} ${
356358
missingKeys.length

0 commit comments

Comments
 (0)