diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json index 9605212f6e2..55f475ddd02 100644 --- a/apps/client/src/translations/en/translation.json +++ b/apps/client/src/translations/en/translation.json @@ -1660,7 +1660,8 @@ "note_context_enabled": "Click to disable note context: {{title}}", "note_context_disabled": "Click to include current note in context", "no_provider_message": "No AI provider configured. Add one to start chatting.", - "add_provider": "Add AI Provider" + "add_provider": "Add AI Provider", + "free": "Free" }, "sidebar_chat": { "title": "AI Chat", @@ -2340,6 +2341,7 @@ "delete_provider_confirmation": "Are you sure you want to delete the provider \"{{name}}\"?", "api_key": "API Key", "api_key_placeholder": "Enter your API key", + "base_url": "Base URL", "cancel": "Cancel", "mcp_title": "MCP (Model Context Protocol)", "mcp_enabled": "MCP server", diff --git a/apps/client/src/widgets/type_widgets/llm_chat/ChatInputBar.tsx b/apps/client/src/widgets/type_widgets/llm_chat/ChatInputBar.tsx index 6491a595b08..7095cc15069 100644 --- a/apps/client/src/widgets/type_widgets/llm_chat/ChatInputBar.tsx +++ b/apps/client/src/widgets/type_widgets/llm_chat/ChatInputBar.tsx @@ -153,7 +153,7 @@ export default function ChatInputBar({ onClick={() => handleModelSelect(model.id)} checked={chat.selectedModel === model.id} > - {model.name} ({model.costDescription}) + {model.name}{model.costDescription && <> ({model.costDescription})} ))} {legacyModels.length > 0 && ( @@ -169,7 +169,7 @@ export default function ChatInputBar({ onClick={() => handleModelSelect(model.id)} checked={chat.selectedModel === model.id} > - {model.name} ({model.costDescription}) + {model.name}{model.costDescription && <> ({model.costDescription})} ))} diff --git a/apps/client/src/widgets/type_widgets/llm_chat/useLlmChat.ts b/apps/client/src/widgets/type_widgets/llm_chat/useLlmChat.ts index 63cbf4bbf42..17cd17eb523 100644 --- a/apps/client/src/widgets/type_widgets/llm_chat/useLlmChat.ts +++ b/apps/client/src/widgets/type_widgets/llm_chat/useLlmChat.ts @@ -3,6 +3,7 @@ import { RefObject } from "preact"; import { useCallback, useEffect, useRef, useState } from "preact/hooks"; import { getAvailableModels, streamChatCompletion } from "../../../services/llm_chat.js"; +import { t } from "../../../services/i18n.js"; import { randomString } from "../../../services/utils.js"; import type { ContentBlock, LlmChatContent, StoredMessage } from "./llm_chat_types.js"; @@ -122,7 +123,11 @@ export function useLlmChat( getAvailableModels().then(models => { const modelsWithDescription = models.map(m => ({ ...m, - costDescription: m.costMultiplier ? `${m.costMultiplier}x` : undefined + costDescription: m.costMultiplier + ? `${m.costMultiplier}x` + : m.pricing.input === 0 && m.pricing.output === 0 + ? t("llm_chat.free") + : undefined })); setAvailableModels(modelsWithDescription); setHasProvider(models.length > 0); diff --git a/apps/client/src/widgets/type_widgets/options/llm/AddProviderModal.tsx b/apps/client/src/widgets/type_widgets/options/llm/AddProviderModal.tsx index 4538cde3b88..755bcd8d645 100644 --- a/apps/client/src/widgets/type_widgets/options/llm/AddProviderModal.tsx +++ b/apps/client/src/widgets/type_widgets/options/llm/AddProviderModal.tsx @@ -11,17 +11,26 @@ export interface LlmProviderConfig { name: string; provider: string; apiKey: string; + /** Base URL for self-hosted providers (e.g. Ollama). */ + baseUrl?: string; } export interface ProviderType { id: string; name: string; + /** Whether this provider needs an API key (defaults to true). */ + needsApiKey?: boolean; + /** Whether this provider needs a base URL. */ + needsBaseUrl?: boolean; + /** Default base URL for the provider. */ + defaultBaseUrl?: string; } export const PROVIDER_TYPES: ProviderType[] = [ { id: "anthropic", name: "Anthropic" }, { id: "openai", name: "OpenAI" }, - { id: "google", name: "Google Gemini" } + { id: "google", name: "Google Gemini" }, + { id: "ollama", name: "Ollama", needsApiKey: false, needsBaseUrl: true, defaultBaseUrl: "http://localhost:11434" } ]; interface AddProviderModalProps { @@ -33,19 +42,34 @@ interface AddProviderModalProps { export default function AddProviderModal({ show, onHidden, onSave }: AddProviderModalProps) { const [selectedProvider, setSelectedProvider] = useState(PROVIDER_TYPES[0].id); const [apiKey, setApiKey] = useState(""); + const [baseUrl, setBaseUrl] = useState(""); const formRef = useRef(null); + const providerType = PROVIDER_TYPES.find(p => p.id === selectedProvider); + const needsApiKey = providerType?.needsApiKey !== false; + const needsBaseUrl = providerType?.needsBaseUrl === true; + + function handleProviderChange(value: string) { + setSelectedProvider(value); + const pt = PROVIDER_TYPES.find(p => p.id === value); + if (pt?.defaultBaseUrl) { + setBaseUrl(pt.defaultBaseUrl); + } else { + setBaseUrl(""); + } + } + function handleSubmit() { - if (!apiKey.trim()) { + if (needsApiKey && !apiKey.trim()) { return; } - const providerType = PROVIDER_TYPES.find(p => p.id === selectedProvider); const newProvider: LlmProviderConfig = { id: `${selectedProvider}_${Date.now()}`, name: providerType?.name || selectedProvider, provider: selectedProvider, - apiKey: apiKey.trim() + apiKey: apiKey.trim(), + ...(needsBaseUrl && baseUrl.trim() ? { baseUrl: baseUrl.trim() } : {}) }; onSave(newProvider); @@ -56,6 +80,7 @@ export default function AddProviderModal({ show, onHidden, onSave }: AddProvider function resetForm() { setSelectedProvider(PROVIDER_TYPES[0].id); setApiKey(""); + setBaseUrl(""); } function handleCancel() { @@ -63,6 +88,8 @@ export default function AddProviderModal({ show, onHidden, onSave }: AddProvider onHidden(); } + const isSubmitDisabled = needsApiKey ? !apiKey.trim() : false; + return createPortal( {t("llm.cancel")} - @@ -89,19 +116,31 @@ export default function AddProviderModal({ show, onHidden, onSave }: AddProvider keyProperty="id" titleProperty="name" currentValue={selectedProvider} - onChange={setSelectedProvider} + onChange={handleProviderChange} /> - - - + {needsApiKey && ( + + + + )} + + {needsBaseUrl && ( + + + + )} , document.body ); diff --git a/apps/server/src/routes/api/llm_chat.ts b/apps/server/src/routes/api/llm_chat.ts index dd5bf149c8b..6ea84b6c453 100644 --- a/apps/server/src/routes/api/llm_chat.ts +++ b/apps/server/src/routes/api/llm_chat.ts @@ -3,6 +3,7 @@ import type { Request, Response } from "express"; import { generateChatTitle } from "../../services/llm/chat_title.js"; import { getAllModels, getProviderByType, hasConfiguredProviders, type LlmProviderConfig } from "../../services/llm/index.js"; +import { OllamaProvider } from "../../services/llm/providers/ollama.js"; import { streamToChunks } from "../../services/llm/stream.js"; import log from "../../services/log.js"; import { safeExtractMessageAndStackFromError } from "../../services/utils.js"; @@ -51,6 +52,12 @@ async function streamChat(req: Request, res: Response) { } const provider = getProviderByType(config.provider || "anthropic"); + + // Ensure Ollama models are loaded so defaultModel/titleModel are set + if (provider instanceof OllamaProvider) { + await provider.loadModels(); + } + const result = provider.chat(messages, config); // Get pricing and display name for the model @@ -90,12 +97,12 @@ async function streamChat(req: Request, res: Response) { /** * Get available models from all configured providers. */ -function getModels(_req: Request, _res: Response) { +async function getModels(_req: Request, _res: Response) { if (!hasConfiguredProviders()) { return { models: [] }; } - return { models: getAllModels() }; + return { models: await getAllModels() }; } export default { diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index 198fa5c22e4..fab968bf560 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -331,7 +331,7 @@ function register(app: express.Application) { // LLM chat endpoints asyncRoute(PST, "/api/llm-chat/stream", [auth.checkApiAuthOrElectron, csrfMiddleware], llmChatRoute.streamChat, null); - apiRoute(GET, "/api/llm-chat/models", llmChatRoute.getModels); + asyncApiRoute(GET, "/api/llm-chat/models", llmChatRoute.getModels); // no CSRF since this is called from android app route(PST, "/api/sender/login", [loginRateLimiter], loginApiRoute.token, apiResultHandler); diff --git a/apps/server/src/services/llm/chat_title.ts b/apps/server/src/services/llm/chat_title.ts index 7350244cae2..fffec5b2d69 100644 --- a/apps/server/src/services/llm/chat_title.ts +++ b/apps/server/src/services/llm/chat_title.ts @@ -1,5 +1,6 @@ import becca from "../../becca/becca.js"; import { getProvider } from "./index.js"; +import { OllamaProvider } from "./providers/ollama.js"; import log from "../log.js"; import { t } from "i18next"; @@ -28,6 +29,12 @@ export async function generateChatTitle(chatNoteId: string, firstMessage: string } const provider = getProvider(); + + // Ensure Ollama models are loaded so titleModel is set + if (provider instanceof OllamaProvider) { + await provider.loadModels(); + } + const title = await provider.generateTitle(firstMessage); if (title) { note.title = title; diff --git a/apps/server/src/services/llm/index.ts b/apps/server/src/services/llm/index.ts index ebf0a066395..b64cdddf3d0 100644 --- a/apps/server/src/services/llm/index.ts +++ b/apps/server/src/services/llm/index.ts @@ -1,6 +1,7 @@ import type { LlmProvider, ModelInfo } from "./types.js"; import { AnthropicProvider } from "./providers/anthropic.js"; import { GoogleProvider } from "./providers/google.js"; +import { OllamaProvider } from "./providers/ollama.js"; import { OpenAiProvider } from "./providers/openai.js"; import optionService from "../options.js"; import log from "../log.js"; @@ -14,13 +15,16 @@ export interface LlmProviderSetup { name: string; provider: string; apiKey: string; + /** Base URL for self-hosted providers (e.g. Ollama). */ + baseUrl?: string; } /** Factory functions for creating provider instances */ -const providerFactories: Record LlmProvider> = { +const providerFactories: Record LlmProvider> = { anthropic: (apiKey) => new AnthropicProvider(apiKey), openai: (apiKey) => new OpenAiProvider(apiKey), - google: (apiKey) => new GoogleProvider(apiKey) + google: (apiKey) => new GoogleProvider(apiKey), + ollama: (_apiKey, baseUrl) => new OllamaProvider(baseUrl) }; /** Cache of instantiated providers by their config ID */ @@ -73,7 +77,7 @@ export function getProvider(providerId?: string): LlmProvider { throw new Error(`Unknown LLM provider type: ${config.provider}. Available: ${Object.keys(providerFactories).join(", ")}`); } - const provider = factory(config.apiKey); + const provider = factory(config.apiKey, config.baseUrl); cachedProviders[config.id] = provider; return provider; } @@ -102,7 +106,7 @@ export function hasConfiguredProviders(): boolean { /** * Get all models from all configured providers, tagged with their provider type. */ -export function getAllModels(): ModelInfo[] { +export async function getAllModels(): Promise { const configs = getConfiguredProviders(); const seenProviderTypes = new Set(); const allModels: ModelInfo[] = []; @@ -116,6 +120,12 @@ export function getAllModels(): ModelInfo[] { try { const provider = getProvider(config.id); + + // Ollama needs to fetch models from the running instance + if (provider instanceof OllamaProvider) { + await provider.loadModels(); + } + const models = provider.getAvailableModels(); for (const model of models) { allModels.push({ ...model, provider: config.provider }); diff --git a/apps/server/src/services/llm/providers/ollama.ts b/apps/server/src/services/llm/providers/ollama.ts new file mode 100644 index 00000000000..87d750ec052 --- /dev/null +++ b/apps/server/src/services/llm/providers/ollama.ts @@ -0,0 +1,156 @@ +/** + * Ollama provider — uses the OpenAI-compatible API that Ollama exposes. + * + * Because Ollama runs locally with a dynamic model list, this provider + * fetches available models from the Ollama instance at runtime instead + * of using a hardcoded list. + */ + +import { createOpenAI, type OpenAIProvider as OpenAISDKProvider } from "@ai-sdk/openai"; + +import log from "../../log.js"; +import { BaseProvider } from "./base_provider.js"; +import type { ModelInfo, ModelPricing } from "../types.js"; + +const DEFAULT_BASE_URL = "http://localhost:11434"; + +/** Ollama models are local and free. */ +const FREE_PRICING: ModelPricing = { input: 0, output: 0 }; + +/** + * Shape of the Ollama `/api/tags` response. + * See https://github.com/ollama/ollama/blob/main/docs/api.md#list-local-models + */ +interface OllamaTagsResponse { + models: Array<{ + name: string; + model: string; + modified_at?: string; + size?: number; + digest?: string; + details?: { + parent_model?: string; + format?: string; + family?: string; + families?: string[]; + parameter_size?: string; + quantization_level?: string; + }; + }>; +} + +/** + * Parse a parameter_size string like "7.6B" or "3.2B" into a number of billions. + * Returns undefined if the string cannot be parsed. + */ +function parseParamSize(paramSize?: string): number | undefined { + if (!paramSize) return undefined; + const match = paramSize.match(/^([\d.]+)\s*([BMK])/i); + if (!match) return undefined; + const value = parseFloat(match[1]); + const unit = match[2].toUpperCase(); + if (unit === "B") return value; + if (unit === "M") return value / 1000; + if (unit === "K") return value / 1_000_000; + return undefined; +} + +/** + * Build a human-readable display name from Ollama model metadata. + * Example: "llama3.2:latest" → "llama3.2:latest (3.2B, Q4_K_M)" + */ +function formatModelName(m: OllamaTagsResponse["models"][number]): string { + const parts: string[] = []; + if (m.details?.parameter_size) { + parts.push(m.details.parameter_size); + } + if (m.details?.quantization_level) { + parts.push(m.details.quantization_level); + } + if (parts.length > 0) { + return `${m.name} (${parts.join(", ")})`; + } + return m.name; +} + +export class OllamaProvider extends BaseProvider { + name = "ollama"; + protected defaultModel = ""; + protected titleModel = ""; + protected availableModels: ModelInfo[] = []; + protected modelPricing: Record = {}; + + private openai: OpenAISDKProvider; + private baseUrl: string; + private modelsLoaded = false; + + constructor(baseUrl?: string) { + super(); + this.baseUrl = baseUrl || DEFAULT_BASE_URL; + + // Ollama exposes an OpenAI-compatible endpoint at /v1 + this.openai = createOpenAI({ + apiKey: "ollama", // Ollama ignores this but the SDK requires it + baseURL: `${this.baseUrl}/v1` + }); + } + + protected createModel(modelId: string) { + return this.openai(modelId); + } + + override getAvailableModels(): ModelInfo[] { + return this.availableModels; + } + + /** + * Fetch available models from the Ollama instance. + * Called by the route handler before returning models to the client. + */ + async loadModels(): Promise { + if (this.modelsLoaded) { + return this.availableModels; + } + + try { + const res = await fetch(`${this.baseUrl}/api/tags`, { + signal: AbortSignal.timeout(5000) + }); + if (!res.ok) { + log.error(`Ollama: failed to fetch models (${res.status})`); + return []; + } + + const data = (await res.json()) as OllamaTagsResponse; + this.availableModels = data.models.map((m, i) => ({ + id: m.name, + name: formatModelName(m), + pricing: FREE_PRICING, + costMultiplier: 0, + isDefault: i === 0 + })); + + this.modelPricing = Object.fromEntries( + this.availableModels.map((m) => [m.id, FREE_PRICING]) + ); + + if (this.availableModels.length > 0) { + this.defaultModel = this.availableModels[0].id; + // Prefer smaller model for titles if available (under 4B params) + const smallModel = data.models.find((m) => { + const size = parseParamSize(m.details?.parameter_size); + return size !== undefined && size < 4; + }) ?? data.models.find((m) => + /small|mini|tiny|phi|gemma.*2b/i.test(m.name) + ); + this.titleModel = smallModel?.name || this.defaultModel; + } + + this.modelsLoaded = true; + return this.availableModels; + } catch (e) { + log.error(`Ollama: failed to connect to ${this.baseUrl}: ${e}`); + return []; + } + } +}