diff --git a/packages/plugins/google/src/api/group.ts b/packages/plugins/google/src/api/group.ts index 87053a337..e01f98ace 100644 --- a/packages/plugins/google/src/api/group.ts +++ b/packages/plugins/google/src/api/group.ts @@ -59,6 +59,41 @@ const AddBundleResponse = Schema.Struct({ toolCount: Schema.Number, }); +const AddServicePayload = Schema.Struct({ + presetId: Schema.String, + slug: Schema.optional(Schema.String), + name: Schema.optional(Schema.String), +}); + +const AddServicesPayload = Schema.Struct({ + services: Schema.Array(AddServicePayload), + baseUrl: Schema.optional(Schema.String), +}); + +const AddServicesResponse = Schema.Struct({ + added: Schema.Array( + Schema.Struct({ + slug: IntegrationSlug, + presetId: Schema.String, + toolCount: Schema.Number, + }), + ), + skipped: Schema.Array( + Schema.Struct({ + slug: IntegrationSlug, + presetId: Schema.String, + reason: Schema.Literal("already_exists"), + }), + ), + failed: Schema.Array( + Schema.Struct({ + slug: IntegrationSlug, + presetId: Schema.String, + error: Schema.String, + }), + ), +}); + const UpdateBundlePayload = Schema.Struct({ urls: Schema.optional(Schema.Array(Schema.String)), }); @@ -103,6 +138,13 @@ export const GoogleGroup = HttpApiGroup.make("google") error: DomainErrors, }), ) + .add( + HttpApiEndpoint.post("addServices", "/google/services", { + payload: AddServicesPayload, + success: AddServicesResponse, + error: DomainErrors, + }), + ) .add( HttpApiEndpoint.get("getIntegration", "/google/integrations/:slug", { params: SlugParams, diff --git a/packages/plugins/google/src/api/handlers.ts b/packages/plugins/google/src/api/handlers.ts index 5f1240467..8dc071f8b 100644 --- a/packages/plugins/google/src/api/handlers.ts +++ b/packages/plugins/google/src/api/handlers.ts @@ -28,6 +28,17 @@ export const GoogleHandlers = HttpApiBuilder.group(ExecutorApiWithGoogle, "googl }), ), ) + .handle("addServices", ({ payload }) => + capture( + Effect.gen(function* () { + const ext = yield* GoogleExtensionService; + return yield* ext.addServices({ + services: payload.services, + baseUrl: payload.baseUrl, + }); + }), + ), + ) .handle("getIntegration", ({ params }) => capture( Effect.gen(function* () { diff --git a/packages/plugins/google/src/react/atoms.ts b/packages/plugins/google/src/react/atoms.ts index b62b6ffed..6e30823bd 100644 --- a/packages/plugins/google/src/react/atoms.ts +++ b/packages/plugins/google/src/react/atoms.ts @@ -19,6 +19,8 @@ export const googleConfigAtom = (slug: IntegrationSlug) => export const addGoogleBundle = GoogleClient.mutation("google", "addBundle"); +export const addGoogleServices = GoogleClient.mutation("google", "addServices"); + export const updateGoogleBundle = GoogleClient.mutation("google", "updateBundle"); export const removeGoogleBundle = GoogleClient.mutation("google", "removeBundle"); diff --git a/packages/plugins/google/src/react/index.ts b/packages/plugins/google/src/react/index.ts index 4ab598dbd..8e3a20856 100644 --- a/packages/plugins/google/src/react/index.ts +++ b/packages/plugins/google/src/react/index.ts @@ -2,6 +2,7 @@ export { googleIntegrationPlugin } from "./source-plugin"; export { GoogleClient } from "./client"; export { addGoogleBundle, + addGoogleServices, googleConfigAtom, googleConfigFamily, googleConfigure, diff --git a/packages/plugins/google/src/sdk/index.ts b/packages/plugins/google/src/sdk/index.ts index ce32c1e1c..d0f3f0ff9 100644 --- a/packages/plugins/google/src/sdk/index.ts +++ b/packages/plugins/google/src/sdk/index.ts @@ -14,6 +14,7 @@ export { GOOGLE_PHOTOS_ICON, GOOGLE_PHOTOS_PRESET_ID, googleStandardUserOAuthPresets, + googleServiceSlug, googleOAuthConsentScopes, googleOAuthConsentScopesForPreset, googleAudienceWarningsForUrls, @@ -34,10 +35,16 @@ export { } from "./oauth-batches"; export { googlePlugin, + type GoogleAddServicesAdded, + type GoogleAddServicesFailed, + type GoogleAddServicesInput, + type GoogleAddServicesResult, + type GoogleAddServicesSkipped, type GoogleBundleConfig, type GoogleConfigureInput, type GooglePluginExtension, type GooglePluginOptions, + type GoogleServiceConfig, type GoogleUpdateInput, type GoogleUpdateResult, } from "./plugin"; diff --git a/packages/plugins/google/src/sdk/plugin.test.ts b/packages/plugins/google/src/sdk/plugin.test.ts index 0bfe0491f..3341eef15 100644 --- a/packages/plugins/google/src/sdk/plugin.test.ts +++ b/packages/plugins/google/src/sdk/plugin.test.ts @@ -431,28 +431,59 @@ const DISCOVERY_BODIES: Readonly> = { [OAUTH2_URL]: toJson(oauth2Doc), }; +interface DiscoveryHttpClientOptions { + readonly failedUrls?: ReadonlySet; + readonly beforeResponse?: (url: string) => Effect.Effect; +} + +const waitOneTurn: Effect.Effect = Effect.promise(() => Promise.resolve()).pipe( + Effect.orDie, +); + // A stub HTTP client that serves the canned Discovery document for whichever // URL the bundle converter fetches. Service-hosted Discovery URLs carry their // version in the query string, so match the full URL before falling back to the // path-only key used by central Discovery URLs. -const discoveryHttpClientLayer = Layer.succeed(HttpClient.HttpClient)( - HttpClient.make((request: HttpClientRequest.HttpClientRequest) => { - const url = new URL(request.url); - const key = `${url.origin}${url.pathname}`; - const body = DISCOVERY_BODIES[url.toString()] ?? DISCOVERY_BODIES[key]; - return Effect.succeed( - HttpClientResponse.fromWeb( +const makeDiscoveryHttpClientLayer = (options: DiscoveryHttpClientOptions = {}) => + Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => { + const url = new URL(request.url); + const key = `${url.origin}${url.pathname}`; + const body = DISCOVERY_BODIES[url.toString()] ?? DISCOVERY_BODIES[key]; + const discoveryUrl = DISCOVERY_BODIES[url.toString()] !== undefined ? url.toString() : key; + const failed = + options.failedUrls?.has(url.toString()) === true || options.failedUrls?.has(key) === true; + const response = HttpClientResponse.fromWeb( request, - body === undefined - ? new Response("not found", { status: 404 }) - : new Response(body, { - status: 200, - headers: { "content-type": "application/json" }, - }), - ), - ); - }), -); + failed + ? new Response("forced failure", { status: 503 }) + : body === undefined + ? new Response("not found", { status: 404 }) + : new Response(body, { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + return (options.beforeResponse?.(discoveryUrl) ?? Effect.void).pipe(Effect.as(response)); + }), + ); + +const discoveryHttpClientLayer = makeDiscoveryHttpClientLayer(); + +const oauthScopesFromConfig = ( + config: { + readonly authenticationTemplate?: readonly { + readonly kind: string; + readonly scopes?: readonly string[]; + }[]; + } | null, +): readonly string[] | undefined => { + const oauth = config?.authenticationTemplate?.find((entry) => entry.kind === "oauth2"); + return oauth?.scopes ? [...oauth.scopes].sort() : undefined; +}; + +const googleDiscoveryToolNames = (toolName: string): readonly string[] => + [toolName, "oauth2.tokeninfo", "oauth2.userinfo.get", "oauth2.userinfo.v2.me.get"].sort(); const bundlePlugins = () => [googlePlugin({ httpClientLayer: discoveryHttpClientLayer }), memoryCredentialsPlugin()] as const; @@ -701,6 +732,238 @@ describe("Google bundle add flow", () => { ); }); +describe("Google per-service add flow", () => { + const serviceExpectations = [ + { + presetId: "google-calendar", + slug: "google_calendar", + name: "Google Calendar", + description: "Calendars, events, ACLs, and scheduling.", + discoveryUrl: CALENDAR_URL, + scopes: ["email", "https://www.googleapis.com/auth/calendar", "openid", "profile"], + toolNames: googleDiscoveryToolNames("calendar.events.list"), + }, + { + presetId: "google-gmail", + slug: "google_gmail", + name: "Gmail", + description: "Messages, threads, labels, and drafts.", + discoveryUrl: GMAIL_URL, + scopes: ["email", "https://mail.google.com/", "openid", "profile"], + toolNames: googleDiscoveryToolNames("gmail.users.messages.list"), + }, + { + presetId: "google-drive", + slug: "google_drive", + name: "Google Drive", + description: "Files, folders, permissions, and shared drives.", + discoveryUrl: DRIVE_URL, + scopes: ["email", "https://www.googleapis.com/auth/drive", "openid", "profile"], + toolNames: googleDiscoveryToolNames("drive.files.list"), + }, + ] as const; + + it.effect( + "addServices creates one integration per preset with exact scopes and health checks", + () => + Effect.scoped( + Effect.gen(function* () { + const executor = yield* createExecutor(makeTestConfig({ plugins: bundlePlugins() })); + const connectedToolNames = (slug: string) => + Effect.gen(function* () { + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make(slug), + integration: IntegrationSlug.make(slug), + template: AuthTemplateSlug.make("googleOAuth2"), + value: "token-xyz", + }); + return (yield* executor.tools.list({ integration: IntegrationSlug.make(slug) })) + .map((tool) => String(tool.name)) + .sort(); + }); + + const result = yield* executor.google.addServices({ + services: serviceExpectations.map((service) => ({ presetId: service.presetId })), + }); + + expect(result).toEqual({ + added: serviceExpectations.map((service) => ({ + slug: IntegrationSlug.make(service.slug), + presetId: service.presetId, + toolCount: 4, + })), + skipped: [], + failed: [], + }); + + const integrations = yield* executor.integrations.list(); + for (const service of serviceExpectations) { + const integration = integrations.find((item) => String(item.slug) === service.slug); + expect(integration?.name).toBe(service.name); + expect(integration?.description).toBe(service.description); + + const config = yield* executor.google.getConfig(service.slug); + expect(config?.googleDiscoveryUrls).toEqual([service.discoveryUrl, OAUTH2_URL]); + expect(oauthScopesFromConfig(config)).toEqual([...service.scopes].sort()); + + const stored = yield* executor.integrations.healthCheck.get( + IntegrationSlug.make(service.slug), + ); + expect(stored?.operation).toBe("oauth2.userinfo.get"); + expect(stored?.args).toBeUndefined(); + expect(stored?.identityField).toBe("email"); + + expect(yield* connectedToolNames(service.slug)).toEqual(service.toolNames); + } + }), + ), + ); + + it.effect( + "addServices isolates a Discovery fetch failure and keeps other services registered", + () => + Effect.scoped( + Effect.gen(function* () { + const failingLayer = makeDiscoveryHttpClientLayer({ + failedUrls: new Set([GMAIL_URL]), + }); + const executor = yield* createExecutor( + makeTestConfig({ + plugins: [googlePlugin({ httpClientLayer: failingLayer }), memoryCredentialsPlugin()], + }), + ); + const connectedToolNames = (slug: string) => + Effect.gen(function* () { + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make(slug), + integration: IntegrationSlug.make(slug), + template: AuthTemplateSlug.make("googleOAuth2"), + value: "token-xyz", + }); + return (yield* executor.tools.list({ integration: IntegrationSlug.make(slug) })) + .map((tool) => String(tool.name)) + .sort(); + }); + + const result = yield* executor.google.addServices({ + services: [ + { presetId: "google-calendar" }, + { presetId: "google-gmail" }, + { presetId: "google-drive" }, + ], + }); + + expect(result).toEqual({ + added: [ + { + slug: IntegrationSlug.make("google_calendar"), + presetId: "google-calendar", + toolCount: 4, + }, + { + slug: IntegrationSlug.make("google_drive"), + presetId: "google-drive", + toolCount: 4, + }, + ], + skipped: [], + failed: [ + { + slug: IntegrationSlug.make("google_gmail"), + presetId: "google-gmail", + error: "Failed to fetch Google Discovery document: HTTP 503", + }, + ], + }); + + expect(yield* executor.google.getIntegration("google_gmail")).toBeNull(); + expect(yield* connectedToolNames("google_calendar")).toEqual( + googleDiscoveryToolNames("calendar.events.list"), + ); + expect(yield* connectedToolNames("google_drive")).toEqual( + googleDiscoveryToolNames("drive.files.list"), + ); + }), + ), + ); + + it.effect("addServices skips an existing service without duplicating integration rows", () => + Effect.scoped( + Effect.gen(function* () { + const executor = yield* createExecutor(makeTestConfig({ plugins: bundlePlugins() })); + + yield* executor.google.addServices({ + services: [{ presetId: "google-calendar" }], + }); + const result = yield* executor.google.addServices({ + services: [{ presetId: "google-calendar" }], + }); + + expect(result).toEqual({ + added: [], + skipped: [ + { + slug: IntegrationSlug.make("google_calendar"), + presetId: "google-calendar", + reason: "already_exists", + }, + ], + failed: [], + }); + + const integrations = yield* executor.integrations.list(); + expect( + integrations.filter((integration) => String(integration.slug) === "google_calendar"), + ).toHaveLength(1); + }), + ), + ); + + it.effect("addServices starts each service Discovery fetch sequentially", () => + Effect.scoped( + Effect.gen(function* () { + let activeServiceFetches = 0; + let maxActiveServiceFetches = 0; + const serviceFetchOrder: string[] = []; + const sequentialLayer = makeDiscoveryHttpClientLayer({ + beforeResponse: (url) => { + if (url === OAUTH2_URL) return Effect.void; + return Effect.gen(function* () { + activeServiceFetches += 1; + maxActiveServiceFetches = Math.max(maxActiveServiceFetches, activeServiceFetches); + serviceFetchOrder.push(url); + yield* waitOneTurn; + activeServiceFetches -= 1; + }); + }, + }); + const executor = yield* createExecutor( + makeTestConfig({ + plugins: [ + googlePlugin({ httpClientLayer: sequentialLayer }), + memoryCredentialsPlugin(), + ], + }), + ); + + const result = yield* executor.google.addServices({ + services: [ + { presetId: "google-calendar" }, + { presetId: "google-gmail" }, + { presetId: "google-drive" }, + ], + }); + + expect(result.failed).toEqual([]); + expect(serviceFetchOrder).toEqual([CALENDAR_URL, GMAIL_URL, DRIVE_URL]); + expect(maxActiveServiceFetches).toBe(1); + }), + ), + ); +}); + describe("Google health-check default", () => { it.effect("default featured bundle auto-configures OAuth2 userinfo identity", () => Effect.scoped( diff --git a/packages/plugins/google/src/sdk/plugin.ts b/packages/plugins/google/src/sdk/plugin.ts index 20dd4a085..19d739c7b 100644 --- a/packages/plugins/google/src/sdk/plugin.ts +++ b/packages/plugins/google/src/sdk/plugin.ts @@ -25,6 +25,7 @@ import { listHealthCheckCandidatesOpenApi, makeDefaultOpenapiStore, normalizeOpenApiAuthInputs, + OpenApiExtractionError, OpenApiParseError, openApiStoredOperationsFromCompiled, resolveOpenApiBackedAnnotations, @@ -43,8 +44,11 @@ import { decodeGoogleIntegrationConfig, type GoogleIntegrationConfig } from "./c import { googleOAuthConsentScopesForPreset, googleOpenApiBundlePreset, + googleOpenApiPresets, googlePhotosOpenApiBundlePreset, googlePhotosOpenApiPresets, + googleServiceSlug, + type GoogleOpenApiPreset, } from "./presets"; const GOOGLE_OAUTH2_DISCOVERY_URL = "https://www.googleapis.com/discovery/v1/apis/oauth2/v2/rest"; @@ -97,6 +101,41 @@ export interface GoogleBundleConfig { readonly baseUrl?: string; } +export interface GoogleServiceConfig { + readonly presetId: string; + readonly slug?: string; + readonly name?: string; +} + +export interface GoogleAddServicesInput { + readonly services: readonly GoogleServiceConfig[]; + readonly baseUrl?: string; +} + +export interface GoogleAddServicesAdded { + readonly slug: IntegrationSlug; + readonly presetId: string; + readonly toolCount: number; +} + +export interface GoogleAddServicesSkipped { + readonly slug: IntegrationSlug; + readonly presetId: string; + readonly reason: "already_exists"; +} + +export interface GoogleAddServicesFailed { + readonly slug: IntegrationSlug; + readonly presetId: string; + readonly error: string; +} + +export interface GoogleAddServicesResult { + readonly added: readonly GoogleAddServicesAdded[]; + readonly skipped: readonly GoogleAddServicesSkipped[]; + readonly failed: readonly GoogleAddServicesFailed[]; +} + export interface GoogleConfigureInput { readonly authenticationTemplate: readonly AuthenticationInput[]; readonly mode?: "merge" | "replace"; @@ -119,6 +158,26 @@ export interface GooglePluginOptions { const DEFAULT_GOOGLE_SLUG = "google"; +const googleOpenApiPresetById: ReadonlyMap = new Map( + googleOpenApiPresets.map((preset) => [preset.id, preset]), +); + +type GoogleAddServiceOutcome = { + readonly added: readonly GoogleAddServicesAdded[]; + readonly skipped: readonly GoogleAddServicesSkipped[]; + readonly failed: readonly GoogleAddServicesFailed[]; +}; + +const googleAddServiceFailure = ( + service: GoogleServiceConfig, + slug: IntegrationSlug, + error: string, +): GoogleAddServiceOutcome => ({ + added: [], + skipped: [], + failed: [{ slug, presetId: service.presetId, error }], +}); + const googlePhotosBundlePresetIdByUrl = new Map( googlePhotosOpenApiPresets.flatMap((preset) => preset.url ? [[normalizeGoogleDiscoveryUrl(preset.url) ?? preset.url, preset.id] as const] : [], @@ -140,6 +199,7 @@ const googlePhotosBundleConsentScopes = ( const fetchGoogleBundleConversion = ( urls: readonly string[], httpClientLayer: Layer.Layer, + consentScopesOverride?: readonly string[], ) => Effect.forEach( urls, @@ -151,10 +211,10 @@ const fetchGoogleBundleConversion = ( { concurrency: 4 }, ).pipe( Effect.flatMap((documents) => { - const consentScopes = googlePhotosBundleConsentScopes(urls); + const consentScopes = consentScopesOverride ?? googlePhotosBundleConsentScopes(urls); return convertGoogleDiscoveryBundleToOpenApi({ documents, - ...(consentScopes ? { consentScopes } : {}), + ...(consentScopes !== undefined ? { consentScopes } : {}), }); }), ); @@ -215,12 +275,23 @@ const makeGooglePluginExtension = ( ) => { const httpClientLayer = options?.httpClientLayer ?? ctx.httpClientLayer; - const addBundle = (config: GoogleBundleConfig) => + const addGoogleOpenApiIntegration = (input: { + readonly urls: readonly string[]; + readonly slug: IntegrationSlug; + readonly name: string; + readonly description: string; + readonly baseUrl?: string; + readonly consentScopes?: readonly string[]; + }) => Effect.gen(function* () { - const urls = yield* googleBundleUrlsWithIdentity(config.urls); - const conversion = yield* fetchGoogleBundleConversion(urls, httpClientLayer); + const urls = yield* googleBundleUrlsWithIdentity(input.urls); + const conversion = yield* fetchGoogleBundleConversion( + urls, + httpClientLayer, + input.consentScopes, + ); const compiled = yield* compileOpenApiSpec(conversion.specText); - const slug = IntegrationSlug.make(config.slug?.trim() || DEFAULT_GOOGLE_SLUG); + const slug = input.slug; const existing = yield* ctx.core.integrations.get(slug); if (existing) { @@ -231,7 +302,7 @@ const makeGooglePluginExtension = ( const integrationConfig: GoogleIntegrationConfig = { specHash, googleDiscoveryUrls: urls, - ...(config.baseUrl ? { baseUrl: config.baseUrl } : {}), + ...(input.baseUrl ? { baseUrl: input.baseUrl } : {}), ...(conversion.authenticationTemplate ? { authenticationTemplate: conversion.authenticationTemplate } : {}), @@ -244,8 +315,8 @@ const makeGooglePluginExtension = ( Effect.gen(function* () { yield* ctx.core.integrations.register({ slug, - name: config.name?.trim() || "Google", - description: config.description ?? "Google APIs", + name: input.name, + description: input.description, config: integrationConfig satisfies GoogleIntegrationConfig as IntegrationConfig, canRemove: true, canRefresh: true, @@ -268,6 +339,92 @@ const makeGooglePluginExtension = ( return { slug, toolCount: compiled.definitions.length }; }); + const addBundle = (config: GoogleBundleConfig) => + addGoogleOpenApiIntegration({ + urls: config.urls, + slug: IntegrationSlug.make(config.slug?.trim() || DEFAULT_GOOGLE_SLUG), + name: config.name?.trim() || "Google", + description: config.description ?? "Google APIs", + baseUrl: config.baseUrl, + }); + + const addOneService = (service: GoogleServiceConfig, baseUrl?: string) => + Effect.gen(function* () { + const preset = googleOpenApiPresetById.get(service.presetId); + if (!preset?.url) { + return yield* new OpenApiParseError({ + message: `Google service preset is not available: ${service.presetId}`, + }); + } + + return yield* addGoogleOpenApiIntegration({ + urls: [preset.url], + slug: IntegrationSlug.make(service.slug?.trim() || googleServiceSlug(service.presetId)), + name: service.name?.trim() || preset.name, + description: preset.summary, + baseUrl, + consentScopes: googleOAuthConsentScopesForPreset(preset.id), + }); + }); + + const addServices = (input: GoogleAddServicesInput) => + Effect.gen(function* () { + const outcomes = yield* Effect.forEach( + input.services, + (service): Effect.Effect => { + const slug = IntegrationSlug.make( + service.slug?.trim() || googleServiceSlug(service.presetId), + ); + return addOneService(service, input.baseUrl).pipe( + Effect.map( + (result): GoogleAddServiceOutcome => ({ + added: [ + { + slug: result.slug, + presetId: service.presetId, + toolCount: result.toolCount, + }, + ], + skipped: [], + failed: [], + }), + ), + Effect.catchTag("IntegrationAlreadyExistsError", () => + Effect.succeed({ + added: [], + skipped: [ + { + slug, + presetId: service.presetId, + reason: "already_exists" as const, + }, + ], + failed: [], + }), + ), + Effect.catchTags({ + OpenApiParseError: (error: OpenApiParseError) => + Effect.succeed(googleAddServiceFailure(service, slug, error.message)), + OpenApiExtractionError: (error: OpenApiExtractionError) => + Effect.succeed(googleAddServiceFailure(service, slug, error.message)), + }), + Effect.catch(() => + Effect.succeed( + googleAddServiceFailure(service, slug, "Failed to add Google service"), + ), + ), + ); + }, + { concurrency: 1 }, + ); + + return { + added: outcomes.flatMap((outcome) => outcome.added), + skipped: outcomes.flatMap((outcome) => outcome.skipped), + failed: outcomes.flatMap((outcome) => outcome.failed), + }; + }); + const updateBundle = (rawSlug: string, input?: GoogleUpdateInput) => Effect.gen(function* () { const slug = IntegrationSlug.make(rawSlug); @@ -333,6 +490,7 @@ const makeGooglePluginExtension = ( return { addBundle, + addServices, updateBundle, removeBundle: (slug: string) => ctx.transaction( diff --git a/packages/plugins/google/src/sdk/presets.ts b/packages/plugins/google/src/sdk/presets.ts index 896326b17..7b7952f11 100644 --- a/packages/plugins/google/src/sdk/presets.ts +++ b/packages/plugins/google/src/sdk/presets.ts @@ -276,6 +276,8 @@ export const googleOAuthConsentScopes: Readonly googleOAuthConsentScopes[presetId] ?? []; +export const googleServiceSlug = (presetId: string): string => presetId.replaceAll("-", "_"); + // --------------------------------------------------------------------------- // Resolve a stored/normalized Discovery URL back to its preset, so a bundled // `google` integration can surface each selected API's `oauthAudience` (e.g. a