feat(cli): mutlisource

Author: maxprilutskiy

.changeset/rich-singers-act.md

@@ -0,0 +1,8 @@
+---
+"@replexica/spec": minor
+"@replexica/cli": minor
+"@replexica/sdk": minor
+"replexica": minor
+---
+
+Add support for multisource localization to the CLI

packages/cli/src/cli/i18n.ts

@@ -9,8 +9,6 @@ import { createLockfileProcessor } from './../workers/lockfile';
 import { createAuthenticator } from './../workers/auth';
 import { ReplexicaEngine } from '@replexica/sdk';
 import { expandPlaceholderedGlob, createBucketLoader } from '../workers/bucket/v2';
-// import fs from 'fs';
-// import path from 'path';
 
 export default new Command()
   .command('i18n')
@@ -110,15 +108,13 @@ export default new Command()
       for (const [bucketType, placeholderedPath] of placeholderedPathsTuples) {
         console.log('');
         const bucketOra = Ora({});
-        bucketOra.info(`Processing ${placeholderedPath}`);
-        // Create the payload processor instance for the current bucket type
-        const sourceBucketFileLoader = createBucketLoader({
+        bucketOra.info(`Processing ${placeholderedPath}`);        
+        // Load the source locale payload
+        const sourcePayload = await createBucketLoader({
           bucketType,
           placeholderedPath,
           locale: i18nConfig.locale.source,
-        });
-        // Load the source locale payload
-        const sourcePayload = await sourceBucketFileLoader.load();
+        }).load();
         // Load saved checksums from the lockfile
         const savedChecksums = await lockfileProcessor.loadChecksums(placeholderedPath);
         // Calculate current checksums for the source payload
@@ -165,12 +161,27 @@ export default new Command()
             try {
               // Use the SDK to localize the payload
               localeOra.start('AI translation in progress...');
+              // Calculate reference payload if specified
+              const referencePayload: any = {};
+              if (i18nConfig.locale.extraSource) {
+                let extraSourcePayload = await createBucketLoader({
+                  bucketType,
+                  placeholderedPath,
+                  locale: i18nConfig.locale.extraSource,
+                }).load();
+                // leave only the keys that are present in the processable payload
+                extraSourcePayload = _.pick(extraSourcePayload, Object.keys(processablePayload));
+                // assign
+                referencePayload[i18nConfig.locale.extraSource] = extraSourcePayload;
+              }
+
               processedPayload = await replexicaEngine.localize(
                 processablePayload,
                 {
                   sourceLocale: i18nConfig.locale.source,
                   targetLocale: targetLocale,
                 },
+                referencePayload,
                 (progress) => {
                   localeOra.text = `(${progress}%) AI translation in progress...`;
                 }

packages/sdk/src/index.ts

@@ -1,8 +1,8 @@
 import Z from 'zod';
-import { sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec';
+import { sourceLocaleSchema, targetLocaleSchema, allLocalesSchema } from '@replexica/spec';
 import { createId } from "@paralleldrive/cuid2";
 
-const replexicaEngineParamsSchema = Z.object({
+const engineParamsSchema = Z.object({
   apiKey: Z.string(),
   apiUrl: Z.string().url().default('https://engine.replexica.com'),
   batchSize: Z.number()
@@ -18,30 +18,35 @@ const replexicaEngineParamsSchema = Z.object({
     .default(200),
 }).passthrough();
 
-const replexicaLocalizationPayloadSchema = Z.record(
+const payloadSchema = Z.record(
   Z.string(),
   Z.any(),
 );
 
-const replexicaLocalizationParamsSchema = Z.object({
+const localizationParamsSchema = Z.object({
   sourceLocale: sourceLocaleSchema,
   targetLocale: targetLocaleSchema,
 });
 
+const referenceSchema = Z.record(
+  allLocalesSchema,
+  payloadSchema,
+);
+
 /**
  * ReplexicaEngine class for interacting with the Replexica API
  */
 export class ReplexicaEngine {
-  private config: Z.infer<typeof replexicaEngineParamsSchema>;
+  private config: Z.infer<typeof engineParamsSchema>;
 
   /**
    * Create a new ReplexicaEngine instance
    * @param config - Configuration options for the Engine
    */
   constructor(
-    config: Partial<Z.infer<typeof replexicaEngineParamsSchema>>,
+    config: Partial<Z.infer<typeof engineParamsSchema>>,
   ) {
-    this.config = replexicaEngineParamsSchema.parse(config);
+    this.config = engineParamsSchema.parse(config);
   }
 
   /**
@@ -52,12 +57,13 @@ export class ReplexicaEngine {
    * @returns Localized content
    */
   async localize(
-    payload: Z.infer<typeof replexicaLocalizationPayloadSchema>,
-    params: Z.infer<typeof replexicaLocalizationParamsSchema>,
+    payload: Z.infer<typeof payloadSchema>,
+    params: Z.infer<typeof localizationParamsSchema>,
+    reference?: Z.infer<typeof referenceSchema>,
     progressCallback?: (progress: number) => void
   ): Promise<Record<string, string>> {
-    const finalPayload = replexicaLocalizationPayloadSchema.parse(payload);
-    const finalParams = replexicaLocalizationParamsSchema.parse(params);
+    const finalPayload = payloadSchema.parse(payload);
+    const finalParams = localizationParamsSchema.parse(params);
 
     const chunkedPayload = this.extractPayloadChunks(finalPayload);
     const processedPayloadChunks: Record<string, string>[] = [];
@@ -74,7 +80,7 @@ export class ReplexicaEngine {
       const processedPayloadChunk = await this.localizeChunk(
         finalParams.sourceLocale,
         finalParams.targetLocale,
-        { meta: {}, data: chunk },
+        { data: chunk, reference },
         workflowId,
       );
       processedPayloadChunks.push(processedPayloadChunk);
@@ -93,7 +99,7 @@ export class ReplexicaEngine {
   private async localizeChunk(
     sourceLocale: string,
     targetLocale: string,
-    payload: { data: any; meta: any; },
+    payload: { data: any; reference: any; },
     workflowId: string,
   ): Promise<Record<string, string>> {
     const res = await fetch(`${this.config.apiUrl}/i18n`, {
@@ -108,8 +114,8 @@ export class ReplexicaEngine {
           source: sourceLocale,
           target: targetLocale,
         },
-        meta: payload.meta,
         data: payload.data,
+        reference: payload.reference,
       }, null, 2),
     });
 

packages/spec/src/config.ts

@@ -1,5 +1,5 @@
 import Z from 'zod';
-import { sourceLocaleSchema, targetLocaleSchema } from './locales';
+import { allLocalesSchema, sourceLocaleSchema, targetLocaleSchema } from './locales';
 import { bucketTypeSchema } from './formats';
 
 // common
@@ -134,9 +134,20 @@ export const configV1_1Definition = extendConfigDefinition(configV1Definition, {
   },
 });
 
+// v1.1 -> v1.2
+// Changes: Add "extraSource" optional field to the locale node of the config
+export const configV1_2Definition = extendConfigDefinition(configV1_1Definition, {
+  createSchema: (baseSchema) => baseSchema.extend({
+    locale: localeSchema.extend({
+      extraSource: allLocalesSchema.optional(),
+    }),
+  }),
+  createDefaultValue: (baseDefaultValue) => baseDefaultValue,
+  createUpgrader: (oldConfig) => oldConfig,
+});
 
 // exports
-const LATEST_CONFIG_DEFINITION = configV1_1Definition;
+const LATEST_CONFIG_DEFINITION = configV1_2Definition;
 
 export type I18nConfig = Z.infer<typeof LATEST_CONFIG_DEFINITION['schema']>;