feat(compiler): google ai support in compiler (#875) (#897)

* feat: update some types to support google as an ai provider

* feat: more type updates for google ai

* feat: change ai logic for more google ai support

* feat: centralize llm error handling logic in lcp/api

* feat: improve llm api key validation and config

* chore: pnpm new

Co-authored-by: Best Codes <106822363+The-Best-Codes@users.noreply.github.com>

Author: maxprilutskiy

.changeset/short-radios-collect.md

@@ -0,0 +1,9 @@
+---
+"@lingo.dev/_compiler": minor
+"@lingo.dev/_react": minor
+"next-app": minor
+"@lingo.dev/_spec": minor
+"lingo.dev": minor
+---
+
+Add support for other providers in the compiler and implement Google AI as a provider.

packages/cli/package.json

@@ -113,6 +113,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@ai-sdk/anthropic": "^1.2.11",
+    "@ai-sdk/google": "^1.2.19",
     "@ai-sdk/openai": "^1.3.22",
     "@babel/generator": "^7.27.1",
     "@babel/parser": "^7.27.1",
@@ -122,10 +123,10 @@
     "@gitbeaker/rest": "^39.34.3",
     "@inkjs/ui": "^2.0.0",
     "@inquirer/prompts": "^7.4.1",
+    "@lingo.dev/_compiler": "workspace:*",
+    "@lingo.dev/_react": "workspace:*",
     "@lingo.dev/_sdk": "workspace:*",
     "@lingo.dev/_spec": "workspace:*",
-    "@lingo.dev/_react": "workspace:*",
-    "@lingo.dev/_compiler": "workspace:*",
     "@modelcontextprotocol/sdk": "^1.5.0",
     "@paralleldrive/cuid2": "^2.2.2",
     "ai": "^4.3.15",

packages/cli/src/cli/localizer/explicit.ts

@@ -1,4 +1,5 @@
 import { createAnthropic } from "@ai-sdk/anthropic";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
 import { createOpenAI } from "@ai-sdk/openai";
 import { I18nConfig } from "@lingo.dev/_spec";
 import chalk from "chalk";
@@ -16,11 +17,11 @@ export default function createExplicitLocalizer(
       throw new Error(
         dedent`
           You're trying to use unsupported provider: ${chalk.dim(provider.id)}.
-        
+
           To fix this issue:
           1. Switch to one of the supported providers, or
           2. Remove the ${chalk.italic("provider")} node from your i18n.json configuration to switch to ${chalk.hex(colors.green)("Lingo.dev")}
-        
+
           ${chalk.hex(colors.blue)("Docs: https://lingo.dev/go/docs")}
         `,
       );
@@ -41,6 +42,15 @@ export default function createExplicitLocalizer(
         apiKeyName: "ANTHROPIC_API_KEY",
         baseUrl: provider.baseUrl,
       });
+    case "google":
+      return createAiSdkLocalizer({
+        factory: (params) =>
+          createGoogleGenerativeAI(params).languageModel(provider.model),
+        id: provider.id,
+        prompt: provider.prompt,
+        apiKeyName: "GOOGLE_API_KEY",
+        baseUrl: provider.baseUrl,
+      });
   }
 }
 

packages/cli/src/cli/processor/index.ts

@@ -7,6 +7,7 @@ import { createBasicTranslator } from "./basic";
 import { createOpenAI } from "@ai-sdk/openai";
 import { colors } from "../constants";
 import { createAnthropic } from "@ai-sdk/anthropic";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
 
 export default function createProcessor(
   provider: I18nConfig["provider"],
@@ -67,6 +68,15 @@ function getPureModelProvider(provider: I18nConfig["provider"]) {
       return createAnthropic({
         apiKey: process.env.ANTHROPIC_API_KEY,
       })(provider.model);
+    case "google":
+      if (!process.env.GOOGLE_API_KEY) {
+        throw new Error(
+          createMissingKeyErrorMessage("Google", "GOOGLE_API_KEY"),
+        );
+      }
+      return createGoogleGenerativeAI({
+        apiKey: process.env.GOOGLE_API_KEY,
+      })(provider.model);
     default:
       throw new Error(createUnsupportedProviderErrorMessage(provider?.id));
   }

packages/cli/src/cli/utils/settings.ts

@@ -36,6 +36,7 @@ export function getSettings(explicitApiKey: string | undefined): CliSettings {
       openaiApiKey: env.OPENAI_API_KEY || systemFile.llm?.openaiApiKey,
       anthropicApiKey: env.ANTHROPIC_API_KEY || systemFile.llm?.anthropicApiKey,
       groqApiKey: env.GROQ_API_KEY || systemFile.llm?.groqApiKey,
+      googleApiKey: env.GOOGLE_API_KEY || systemFile.llm?.googleApiKey,
     },
   };
 }
@@ -68,6 +69,7 @@ const SettingsSchema = Z.object({
     openaiApiKey: Z.string().optional(),
     anthropicApiKey: Z.string().optional(),
     groqApiKey: Z.string().optional(),
+    googleApiKey: Z.string().optional(),
   }),
 });
 
@@ -96,6 +98,7 @@ function _loadEnv() {
     OPENAI_API_KEY: Z.string().optional(),
     ANTHROPIC_API_KEY: Z.string().optional(),
     GROQ_API_KEY: Z.string().optional(),
+    GOOGLE_API_KEY: Z.string().optional(),
   })
     .passthrough()
     .parse(process.env);
@@ -118,6 +121,7 @@ function _loadSystemFile() {
       openaiApiKey: Z.string().optional(),
       anthropicApiKey: Z.string().optional(),
       groqApiKey: Z.string().optional(),
+      googleApiKey: Z.string().optional(),
     }).optional(),
   })
     .passthrough()
@@ -182,6 +186,12 @@ function _envVarsInfo() {
       `ℹ️  Using GROQ_API_KEY env var instead of key from user config`,
     );
   }
+  if (env.GOOGLE_API_KEY && systemFile.llm?.googleApiKey) {
+    console.info(
+      "\x1b[36m%s\x1b[0m",
+      `ℹ️  Using GOOGLE_API_KEY env var instead of key from user config`,
+    );
+  }
   if (env.LINGODOTDEV_API_URL) {
     console.info(
       "\x1b[36m%s\x1b[0m",

packages/compiler/package.json

@@ -36,6 +36,7 @@
     "typescript": "^5.4.5"
   },
   "dependencies": {
+    "@ai-sdk/google": "^1.2.19",
     "@ai-sdk/groq": "^1.2.3",
     "@babel/generator": "^7.26.5",
     "@babel/parser": "^7.26.7",

packages/compiler/src/index.ts

@@ -32,19 +32,42 @@ import { LCP_DICTIONARY_FILE_NAME } from "./_const";
 import { LCPCache } from "./lib/lcp/cache";
 import { getInvalidLocales } from "./utils/locales";
 import { clientDictionaryLoaderMutation } from "./client-dictionary-loader";
-import { getGroqKeyFromEnv, getGroqKeyFromRc } from "./utils/groq";
+import {
+  getGroqKeyFromEnv,
+  getGroqKeyFromRc,
+  getGoogleKeyFromEnv,
+  getGoogleKeyFromRc,
+} from "./utils/llm-api-key";
 import { isRunningInCIOrDocker } from "./utils/env";
+import { providerDetails } from "./lib/lcp/api/provider-details";
+
+const keyCheckers: Record<
+  string,
+  {
+    checkEnv: () => string | undefined;
+    checkRc: () => string | undefined;
+  }
+> = {
+  groq: {
+    checkEnv: getGroqKeyFromEnv,
+    checkRc: getGroqKeyFromRc,
+  },
+  google: {
+    checkEnv: getGoogleKeyFromEnv,
+    checkRc: getGoogleKeyFromRc,
+  },
+};
 
 const unplugin = createUnplugin<Partial<typeof defaultParams> | undefined>(
   (_params, _meta) => {
     console.log("ℹ️  Starting Lingo.dev compiler...");
 
+    const params = _.defaults(_params, defaultParams);
+
     // Validate if not in CI or Docker
     if (!isRunningInCIOrDocker()) {
-      validateGroqKeyDetails();
+      validateLLMKeyDetails(params.models);
     }
-    // Continue
-    const params = _.defaults(_params, defaultParams);
 
     const invalidLocales = getInvalidLocales(
       params.models,
@@ -60,7 +83,7 @@ const unplugin = createUnplugin<Partial<typeof defaultParams> | undefined>(
         1. Refer to documentation for help: https://docs.lingo.dev/
         2. If you want to use a different LLM, raise an issue in our open-source repo: https://lingo.dev/go/gh
         3. If you have questions, feature requests, or would like to contribute, join our Discord: https://lingo.dev/go/discord
-  
+
         ✨
       `);
       process.exit(1);
@@ -183,60 +206,109 @@ export default {
 };
 
 /**
- * Print helpful information about where the GROQ API key was discovered.
- * The compiler looks for the key first in the environment (incl. .env files)
- * and then in the user-wide configuration. Environment always wins.
+ * Print helpful information about where the LLM API keys for configured providers
+ * were discovered. The compiler looks for the key first in the environment
+ * (incl. .env files) and then in the user-wide configuration. Environment always wins.
+ * @param models The locale to model mapping configuration.
  */
-function validateGroqKeyDetails(): void {
-  const groq = {
-    fromEnv: getGroqKeyFromEnv(),
-    fromRc: getGroqKeyFromRc(),
-  };
+function validateLLMKeyDetails(models: Record<string, string>): void {
+  const configuredProviders = _.chain(Object.values(models))
+    .map((modelString) => modelString.split(":")[0]) // Extract provider ID
+    .filter(Boolean) // Remove empty strings
+    .uniq() // Get unique providers
+    .filter(
+      (providerId) =>
+        providerDetails.hasOwnProperty(providerId) &&
+        keyCheckers.hasOwnProperty(providerId),
+    ) // Only check for known and implemented providers
+    .value();
+
+  if (configuredProviders.length === 0) {
+    // No LLM providers configured that we can validate keys for.
+    return;
+  }
+
+  const keyStatuses: Record<
+    string,
+    {
+      foundInEnv: boolean;
+      foundInRc: boolean;
+      details: (typeof providerDetails)[string];
+    }
+  > = {};
+  const missingProviders: string[] = [];
+  const foundProviders: string[] = [];
+
+  for (const providerId of configuredProviders) {
+    const details = providerDetails[providerId];
+    const checkers = keyCheckers[providerId];
+    if (!details || !checkers) continue; // Should not happen due to filter above
+
+    const foundInEnv = checkers.checkEnv() !== undefined;
+    const foundInRc = checkers.checkRc() !== undefined;
+
+    keyStatuses[providerId] = { foundInEnv, foundInRc, details };
+
+    if (!foundInEnv && !foundInRc) {
+      missingProviders.push(providerId);
+    } else {
+      foundProviders.push(providerId);
+    }
+  }
 
-  if (!groq.fromEnv && !groq.fromRc) {
+  if (missingProviders.length > 0) {
     console.log(dedent`
       \n
-      💡 You're using Lingo.dev Localization Compiler in your project, which requires a GROQ API key to work.
+      💡 Lingo.dev Localization Compiler is configured to use the following LLM provider(s): ${configuredProviders.join(", ")}.
+
+      The compiler requires API keys for these providers to work, but the following keys are missing:
+    `);
+
+    for (const providerId of missingProviders) {
+      const status = keyStatuses[providerId];
+      if (!status) continue;
+      console.log(dedent`
+          ⚠️  ${status.details.name} API key is missing. Set ${status.details.apiKeyEnvVar} environment variable.
 
-      👉 You can set the API key in one of the following ways:
-      1. User-wide: Run npx lingo.dev@latest config set llm.groqApiKey <your-api-key>
-      2. Project-wide: Add GROQ_API_KEY=<your-api-key> to .env file in every project that uses Lingo.dev Localization Compiler
-      3. Session-wide: Run export GROQ_API_KEY=<your-api-key> in your terminal before running the compiler to set the API key for the current session
+          👉 You can set the API key in one of the following ways:
+          1. User-wide: Run npx lingo.dev@latest config set ${status.details.apiKeyConfigKey || "<config-key-not-available>"} <your-api-key>
+          2. Project-wide: Add ${status.details.apiKeyEnvVar}=<your-api-key> to .env file in every project that uses Lingo.dev Localization Compiler
+          3. Session-wide: Run export ${status.details.apiKeyEnvVar}=<your-api-key> in your terminal before running the compiler to set the API key for the current session
 
+          ⭐️ If you don't yet have a ${status.details.name} API key, get one for free at ${status.details.getKeyLink}
+        `);
+    }
+
+    console.log(dedent`
+      \n
       ⭐️ Also:
-      1. If you don't yet have a GROQ API key, get one for free at https://groq.com
-      2. If you want to use a different LLM, raise an issue in our open-source repo: https://lingo.dev/go/gh
+      1. If you want to use a different LLM, update your configuration. Refer to documentation for help: https://docs.lingo.dev/
+      2. If the model/provider you want to use isn't supported yet, raise an issue in our open-source repo: https://lingo.dev/go/gh
       3. If you have questions, feature requests, or would like to contribute, join our Discord: https://lingo.dev/go/discord
 
       ✨
     `);
     process.exit(1);
-  } else if (groq.fromEnv && groq.fromRc) {
-    console.log(
-      dedent`
-        🔑  GROQ API key detected in both environment variables and your user-wide configuration.
-
-        👉  The compiler will use the key from the environment because it has higher priority.
-
-        • To update the user-wide key run: npx lingo.dev@latest config set llm.groqApiKey <your-api-key>
-        • To remove it run: npx lingo.dev@latest config unset llm.groqApiKey
-        • To remove the env variable from the current session run: unset GROQ_API_KEY
-      `,
-    );
-  } else if (groq.fromEnv && !groq.fromRc) {
-    console.log(
-      dedent`
-        🔑  GROQ API key loaded from environment variables.
-
-        • You can also save the key user-wide with: npx lingo.dev@latest config set llm.groqApiKey <your-api-key>
-        • Or remove the env variable from the current session with: unset GROQ_API_KEY
-      `,
-    );
-  } else if (!groq.fromEnv && groq.fromRc) {
-    console.log(
-      dedent`
-        🔑  GROQ API key loaded from your user-wide configuration.
-      `,
-    );
+  } else if (foundProviders.length > 0) {
+    console.log(dedent`
+        \n
+        🔑  LLM API keys detected for configured providers: ${foundProviders.join(", ")}.
+      `);
+    for (const providerId of foundProviders) {
+      const status = keyStatuses[providerId];
+      if (!status) continue;
+      let sourceMessage = "";
+      if (status.foundInEnv && status.foundInRc) {
+        sourceMessage = `from both environment variables (${status.details.apiKeyEnvVar}) and your user-wide configuration. The key from the environment will be used because it has higher priority.`;
+      } else if (status.foundInEnv) {
+        sourceMessage = `from environment variables (${status.details.apiKeyEnvVar}).`;
+      } else if (status.foundInRc) {
+        sourceMessage = `from your user-wide configuration${status.details.apiKeyConfigKey ? ` (${status.details.apiKeyConfigKey})` : ""}.`;
+      }
+      console.log(dedent`
+          • ${status.details.name} API key loaded ${sourceMessage}
+        `);
+    }
+    console.log("✨");
   }
 }