|
| 1 | +import { expect } from '@playwright/test' |
| 2 | +import type { Page, Route } from '@playwright/test' |
| 3 | + |
| 4 | +import type { RemoteConfig } from '@/platform/remoteConfig/types' |
| 5 | + |
| 6 | +import { comfyPageFixture as test } from '@e2e/fixtures/ComfyPage' |
| 7 | +import { bootCloud, mockCloudBoot } from '@e2e/fixtures/utils/cloudBootMocks' |
| 8 | +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' |
| 9 | + |
| 10 | +/** |
| 11 | + * End-to-end coverage for the user-secrets (API keys) surface in the cloud app: |
| 12 | + * add a provider key, see it listed, delete it — the full CRUD round-trip — |
| 13 | + * plus the entitlement contract that a non-entitled account never sees the |
| 14 | + * gated providers. |
| 15 | + * |
| 16 | + * Drives a raw `page` against fully-mocked endpoints (the `comfyPage` fixture |
| 17 | + * would reach the OSS devtools backend during setup); `mockCloudBoot` + |
| 18 | + * `bootCloud` boot the app signed-in, and this spec layers a stateful in-memory |
| 19 | + * `/secrets` backend on top so the flow is deterministic and never touches a |
| 20 | + * real server. |
| 21 | + */ |
| 22 | +const APP_URL = process.env.PLAYWRIGHT_TEST_URL || 'http://localhost:8188' |
| 23 | + |
| 24 | +// `/api/features` is the remote-config source. Enabling user secrets is what |
| 25 | +// surfaces the Secrets settings panel for a signed-in user. |
| 26 | +const BOOT_FEATURES = { |
| 27 | + user_secrets_enabled: true |
| 28 | +} satisfies RemoteConfig |
| 29 | + |
| 30 | +// TutorialCompleted suppresses the new-user template browser, whose modal |
| 31 | +// overlay (z-1700) would otherwise intercept clicks on the settings dialog. |
| 32 | +const BOOT_SETTINGS = { 'Comfy.TutorialCompleted': true } |
| 33 | + |
| 34 | +// The plaintext key a user types in. It must be sent on create but NEVER echoed |
| 35 | +// back by the API or rendered anywhere in the UI. |
| 36 | +const RUNWAY_KEY_VALUE = 'sk-runway-do-not-echo-0xDEADBEEF' |
| 37 | + |
| 38 | +interface SecretRecord { |
| 39 | + id: string |
| 40 | + name: string |
| 41 | + provider?: string |
| 42 | + created_at: string |
| 43 | + updated_at: string |
| 44 | + last_used_at?: string |
| 45 | +} |
| 46 | + |
| 47 | +interface CreateCapture { |
| 48 | + name?: string |
| 49 | + provider?: string |
| 50 | + secret_value?: string |
| 51 | +} |
| 52 | + |
| 53 | +interface SecretsBackend { |
| 54 | + /** Bodies received by POST /secrets, in order — for asserting what was sent. */ |
| 55 | + createRequests: CreateCapture[] |
| 56 | + /** Current server-side store — for asserting delete actually removed a row. */ |
| 57 | + store: SecretRecord[] |
| 58 | +} |
| 59 | + |
| 60 | +/** |
| 61 | + * Stateful mock of the ingest `/secrets` surface. A single route handler |
| 62 | + * branches on path + method so registration order can never make a specific |
| 63 | + * path (`/secrets/providers`, `/secrets/:id`) lose to the collection glob. |
| 64 | + * |
| 65 | + * `providerIds` models entitlement: an entitled account sees runway/gemini, |
| 66 | + * a non-entitled account gets an empty list (the server omits them). |
| 67 | + */ |
| 68 | +async function mockSecretsBackend( |
| 69 | + page: Page, |
| 70 | + providerIds: string[] |
| 71 | +): Promise<SecretsBackend> { |
| 72 | + const backend: SecretsBackend = { createRequests: [], store: [] } |
| 73 | + let idSeq = 0 |
| 74 | + |
| 75 | + const respondList = (route: Route) => |
| 76 | + route.fulfill(jsonRoute({ data: backend.store })) |
| 77 | + |
| 78 | + await page.route('**/api/secrets**', async (route) => { |
| 79 | + const request = route.request() |
| 80 | + const { pathname } = new URL(request.url()) |
| 81 | + const method = request.method() |
| 82 | + |
| 83 | + // The glob `**/api/secrets**` also matches the panel's own lazy-loaded |
| 84 | + // source module (`/src/platform/secrets/api/secretsApi.ts`), whose path |
| 85 | + // contains the `/api/secrets` substring. Fulfilling that dev-server module |
| 86 | + // request with JSON breaks the dynamic import and the panel never mounts. |
| 87 | + // Anchor to the start of the pathname so only genuine `/api/secrets…` API |
| 88 | + // routes are handled; everything else falls through to the real Vite server. |
| 89 | + if (!/^\/api\/secrets(\/|$)/.test(pathname)) { |
| 90 | + return route.continue() |
| 91 | + } |
| 92 | + |
| 93 | + // GET /secrets/providers — the entitlement-gated provider allowlist. |
| 94 | + if (pathname.endsWith('/secrets/providers')) { |
| 95 | + return route.fulfill( |
| 96 | + jsonRoute({ data: providerIds.map((id) => ({ id })) }) |
| 97 | + ) |
| 98 | + } |
| 99 | + |
| 100 | + // /secrets/:id — item routes (only DELETE is exercised by this flow). |
| 101 | + const itemMatch = pathname.match(/\/secrets\/([^/]+)$/) |
| 102 | + if (itemMatch) { |
| 103 | + const id = itemMatch[1] |
| 104 | + if (method === 'DELETE') { |
| 105 | + backend.store = backend.store.filter((s) => s.id !== id) |
| 106 | + return route.fulfill({ status: 204, body: '' }) |
| 107 | + } |
| 108 | + return respondList(route) |
| 109 | + } |
| 110 | + |
| 111 | + // /secrets — collection routes. |
| 112 | + if (method === 'POST') { |
| 113 | + const body = (request.postDataJSON() ?? {}) as CreateCapture |
| 114 | + backend.createRequests.push(body) |
| 115 | + idSeq += 1 |
| 116 | + const created: SecretRecord = { |
| 117 | + id: `00000000-0000-4000-8000-${String(idSeq).padStart(12, '0')}`, |
| 118 | + name: body.name ?? '', |
| 119 | + provider: body.provider, |
| 120 | + created_at: '2026-07-08T00:00:00Z', |
| 121 | + updated_at: '2026-07-08T00:00:00Z' |
| 122 | + } |
| 123 | + backend.store.push(created) |
| 124 | + // Response echoes metadata ONLY — the schema has no secret_value field. |
| 125 | + return route.fulfill(jsonRoute(created)) |
| 126 | + } |
| 127 | + |
| 128 | + // GET /secrets (list). |
| 129 | + return respondList(route) |
| 130 | + }) |
| 131 | + |
| 132 | + return backend |
| 133 | +} |
| 134 | + |
| 135 | +/** |
| 136 | + * Open the settings dialog and land on the Secrets panel, waiting for both the |
| 137 | + * provider allowlist and the secret list to resolve so subsequent assertions |
| 138 | + * are not racing the panel's on-mount fetches. |
| 139 | + */ |
| 140 | +async function openSecretsPanel(page: Page) { |
| 141 | + const settingsDialog = page.getByTestId('settings-dialog') |
| 142 | + |
| 143 | + await page.evaluate(() => { |
| 144 | + const app = window.app |
| 145 | + if (!app) throw new Error('window.app is not available') |
| 146 | + return app.extensionManager.command.execute('Comfy.ShowSettingsDialog') |
| 147 | + }) |
| 148 | + await settingsDialog.waitFor({ state: 'visible' }) |
| 149 | + |
| 150 | + const providersResolved = page.waitForResponse((r) => |
| 151 | + r.url().includes('/api/secrets/providers') |
| 152 | + ) |
| 153 | + const listResolved = page.waitForResponse( |
| 154 | + (r) => |
| 155 | + /\/api\/secrets(\?|$)/.test(r.url()) && r.request().method() === 'GET' |
| 156 | + ) |
| 157 | + |
| 158 | + await settingsDialog |
| 159 | + .locator('nav') |
| 160 | + .getByRole('button', { name: 'Secrets' }) |
| 161 | + .click() |
| 162 | + |
| 163 | + await Promise.all([providersResolved, listResolved]) |
| 164 | + return settingsDialog |
| 165 | +} |
| 166 | + |
| 167 | +test.describe('Cloud user secrets (API keys)', { tag: '@cloud' }, () => { |
| 168 | + test('an entitled account can add, list, and delete a provider key', async ({ |
| 169 | + page |
| 170 | + }) => { |
| 171 | + test.slow() |
| 172 | + |
| 173 | + await mockCloudBoot(page, { |
| 174 | + features: BOOT_FEATURES, |
| 175 | + settings: BOOT_SETTINGS |
| 176 | + }) |
| 177 | + await bootCloud(page) |
| 178 | + const backend = await mockSecretsBackend(page, ['runway', 'gemini']) |
| 179 | + |
| 180 | + await page.goto(APP_URL) |
| 181 | + await page.waitForFunction(() => !!window.app?.extensionManager, null, { |
| 182 | + timeout: 45_000 |
| 183 | + }) |
| 184 | + |
| 185 | + const settingsDialog = await openSecretsPanel(page) |
| 186 | + |
| 187 | + // Empty state before anything is added. |
| 188 | + await expect(settingsDialog.getByText(/No secrets stored/)).toBeVisible() |
| 189 | + |
| 190 | + // --- ADD ------------------------------------------------------------- |
| 191 | + await settingsDialog.getByRole('button', { name: 'Add Secret' }).click() |
| 192 | + |
| 193 | + const formDialog = page |
| 194 | + .getByRole('dialog') |
| 195 | + .filter({ hasText: 'Secret Value' }) |
| 196 | + await expect(formDialog).toBeVisible() |
| 197 | + |
| 198 | + // Pick the entitled Runway provider from the server-driven dropdown. |
| 199 | + await formDialog.locator('#secret-provider').click() |
| 200 | + await page.getByRole('option', { name: 'Runway' }).click() |
| 201 | + |
| 202 | + await formDialog.locator('#secret-name').fill('My Runway Key') |
| 203 | + await formDialog.locator('input[type="password"]').fill(RUNWAY_KEY_VALUE) |
| 204 | + |
| 205 | + await formDialog.getByRole('button', { name: 'Save', exact: true }).click() |
| 206 | + await expect(formDialog).toBeHidden() |
| 207 | + |
| 208 | + // --- LIST ------------------------------------------------------------ |
| 209 | + await expect(settingsDialog.getByText('My Runway Key')).toBeVisible() |
| 210 | + await expect(settingsDialog.getByText(/No secrets stored/)).toBeHidden() |
| 211 | + |
| 212 | + // The create request carried the plaintext value + provider... |
| 213 | + expect(backend.createRequests).toHaveLength(1) |
| 214 | + expect(backend.createRequests[0]).toMatchObject({ |
| 215 | + name: 'My Runway Key', |
| 216 | + provider: 'runway', |
| 217 | + secret_value: RUNWAY_KEY_VALUE |
| 218 | + }) |
| 219 | + // ...but the value must never be echoed back into the list — the API |
| 220 | + // response carries metadata only, so nothing should render it as text. |
| 221 | + await expect(page.getByText(RUNWAY_KEY_VALUE)).toHaveCount(0) |
| 222 | + |
| 223 | + // --- DELETE ---------------------------------------------------------- |
| 224 | + await settingsDialog |
| 225 | + .getByRole('button', { name: 'Delete', exact: true }) |
| 226 | + .click() |
| 227 | + |
| 228 | + const confirmDialog = page |
| 229 | + .getByRole('dialog') |
| 230 | + .filter({ hasText: 'Delete Secret' }) |
| 231 | + await confirmDialog |
| 232 | + .getByRole('button', { name: 'Delete', exact: true }) |
| 233 | + .click() |
| 234 | + |
| 235 | + await expect(settingsDialog.getByText('My Runway Key')).toBeHidden() |
| 236 | + await expect(settingsDialog.getByText(/No secrets stored/)).toBeVisible() |
| 237 | + expect(backend.store).toHaveLength(0) |
| 238 | + }) |
| 239 | + |
| 240 | + test('a non-entitled account never sees the gated providers', async ({ |
| 241 | + page |
| 242 | + }) => { |
| 243 | + test.slow() |
| 244 | + |
| 245 | + await mockCloudBoot(page, { |
| 246 | + features: BOOT_FEATURES, |
| 247 | + settings: BOOT_SETTINGS |
| 248 | + }) |
| 249 | + await bootCloud(page) |
| 250 | + // Non-entitled: the server omits runway/gemini from the allowlist. |
| 251 | + await mockSecretsBackend(page, []) |
| 252 | + |
| 253 | + await page.goto(APP_URL) |
| 254 | + await page.waitForFunction(() => !!window.app?.extensionManager, null, { |
| 255 | + timeout: 45_000 |
| 256 | + }) |
| 257 | + |
| 258 | + const settingsDialog = await openSecretsPanel(page) |
| 259 | + await expect(settingsDialog.getByText(/No secrets stored/)).toBeVisible() |
| 260 | + |
| 261 | + // The add form opens, but its provider dropdown is empty — the gated |
| 262 | + // providers must not appear anywhere. |
| 263 | + await settingsDialog.getByRole('button', { name: 'Add Secret' }).click() |
| 264 | + const formDialog = page |
| 265 | + .getByRole('dialog') |
| 266 | + .filter({ hasText: 'Secret Value' }) |
| 267 | + await expect(formDialog).toBeVisible() |
| 268 | + |
| 269 | + await formDialog.locator('#secret-provider').click() |
| 270 | + // Anchor on the opened listbox so the absence assertions below can't pass |
| 271 | + // vacuously against a dropdown that never opened. |
| 272 | + const providerListbox = page.getByRole('listbox') |
| 273 | + await expect(providerListbox).toBeVisible() |
| 274 | + // An empty allowlist must yield an empty dropdown. Asserting zero options |
| 275 | + // (not just runway/gemini absent) also rejects the fetch-failure fallback, |
| 276 | + // where `availableProviders` is null and the default providers would show. |
| 277 | + await expect(providerListbox.getByRole('option')).toHaveCount(0) |
| 278 | + }) |
| 279 | +}) |
0 commit comments