fix(serenity): add a server-side kill switch for async provisioning (LLMO-7418 external-review Finding 15)

Previously a caller passing `async: true` got the async provisioning path
unconditionally, with no way to disable it org-wide short of a code change
reverting every call site at once.

Adds isAsyncProvisioningKillSwitched (serenity-active.js), reusing the same
cached feature-flag machinery as every other predicate in that file, gated
on a new opt-OUT flag (serenity_async_provisioning_disabled — off by
default, so a transient read failure never silently disables async
provisioning). Wired in immediately before validateAsync at all 6 async
provisioning call sites (createMarket, createBrandForOrg's two branches,
activate's three branches); a caller hits 503 and falls back to the
synchronous path on retry when the switch is on.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: Iris Alexandrescu

src/controllers/brands.js

@@ -98,6 +98,7 @@ import {
   isSerenityActiveForBrand,
   isSerenityActiveForOrg,
   isSerenityUiActiveForOrg,
+  isAsyncProvisioningKillSwitched,
 } from '../support/serenity/serenity-active.js';
 import {
   buildReservedIdentities,
@@ -1870,6 +1871,18 @@ function BrandsController(ctx, log, env) {
           // is NOT permanent: the synchronous branch is the LLMO-7352 bug pattern itself, slated
           // for removal once every known caller has migrated to `async: true`.
           if (validateAsync(brandData)) {
+            // LLMO-7418 external-review Finding 15: server-side kill switch — lets ops disable
+            // the async path for this organization without a deploy if it misbehaves in
+            // production. The caller falls back to the synchronous path on its own retry.
+            if (await isAsyncProvisioningKillSwitched(context, spaceCatId, log)) {
+              return createResponse(
+                {
+                  error: 'asyncProvisioningDisabled',
+                  message: 'Async provisioning is temporarily disabled for this organization; retry without async: true',
+                },
+                503,
+              );
+            }
             // brandAliases/urls/competitors are NOT read here (unlike the sync branch below): the
             // brand row this section persists below (upsertBrand) writes them to storage, and the
             // async chain's orchestration reads them back from there — the same DB-backed source
@@ -1948,6 +1961,17 @@ function BrandsController(ctx, log, env) {
           // sub-workspace provisioning off to provision-workspace-job — no chained job, since a
           // bare create has no project to create once the workspace is ready.
           //
+          // LLMO-7418 external-review Finding 15: server-side kill switch — see the
+          // hasSemrushMarket branch above for rationale.
+          if (await isAsyncProvisioningKillSwitched(context, spaceCatId, log)) {
+            return createResponse(
+              {
+                error: 'asyncProvisioningDisabled',
+                message: 'Async provisioning is temporarily disabled for this organization; retry without async: true',
+              },
+              503,
+            );
+          }
           // Resolved and validated HERE, before any write — same rationale as the
           // hasSemrushMarket branch: a missing org workspace config must never leave a
           // persisted, permanently-inert brand row behind.

src/controllers/serenity.js

@@ -92,7 +92,10 @@ import {
   handleTagImpactSubworkspace,
 } from '../support/serenity/handlers/tags.js';
 import { ensureSubworkspace, decommissionBrandWorkspace } from '../support/serenity/workspace-lifecycle.js';
-import { isSerenityActiveForBrand } from '../support/serenity/serenity-active.js';
+import {
+  isSerenityActiveForBrand,
+  isAsyncProvisioningKillSwitched,
+} from '../support/serenity/serenity-active.js';
 import { marketForGeoTargetId } from '../support/serenity/locations.js';
 import { brandNeedles, classifyBrandedTag } from '../support/serenity/branded-classifier.js';
 import { computeWriteDeadline } from '../support/serenity/intent-classification.js';
@@ -1045,6 +1048,18 @@ function SerenityController(context, log, env) {
         // synchronous branch is the LLMO-7352 bug pattern itself, not a valid alternative, and
         // is slated for removal once every known caller has migrated to `async: true`.
         if (validateAsync(requestBody)) {
+          // LLMO-7418 external-review Finding 15: server-side kill switch — lets ops disable
+          // the async path for this organization without a deploy if it misbehaves in
+          // production. The caller falls back to the synchronous path on its own retry.
+          if (await isAsyncProvisioningKillSwitched(ctx, ctx?.params?.spaceCatId, log)) {
+            return createResponse(
+              {
+                error: 'asyncProvisioningDisabled',
+                message: 'Async provisioning is temporarily disabled for this organization; retry without async: true',
+              },
+              503,
+            );
+          }
           // The worker's existing-pointer fast path (provision-workspace-job.js) polls THIS
           // brand's already-canonical workspace rather than provisioning a new one — every
           // brand reaching this branch already has one (`auth.mode === 'subworkspace'` IS that
@@ -1641,6 +1656,17 @@ function SerenityController(context, log, env) {
         // `provision-workspace-job` ->
         // `serenity-activate-brand-workspace` job chain instead.
         if (validateAsync(body)) {
+          // LLMO-7418 external-review Finding 15: server-side kill switch — see createMarket's
+          // async branch for the full rationale.
+          if (await isAsyncProvisioningKillSwitched(ctx, ctx?.params?.spaceCatId, log)) {
+            return createResponse(
+              {
+                error: 'asyncProvisioningDisabled',
+                message: 'Async provisioning is temporarily disabled for this organization; retry without async: true',
+              },
+              503,
+            );
+          }
           // LLMO-7418 external-review Finding 9: see createMarket's async branch for the full
           // rationale — reconcile a stale in-flight attempt (reusing the sync guard's own logic)
           // before minting a new one, since beginProvisioningAttempt's own CAS has no staleness
@@ -1779,6 +1805,17 @@ function SerenityController(context, log, env) {
         // activate-brand-workspace-job.js's save-divergence handling matches this branch's own
         // 207-not-502 contract.
         if (validateAsync(body)) {
+          // LLMO-7418 external-review Finding 15: server-side kill switch — see createMarket's
+          // async branch for the full rationale.
+          if (await isAsyncProvisioningKillSwitched(ctx, ctx?.params?.spaceCatId, log)) {
+            return createResponse(
+              {
+                error: 'asyncProvisioningDisabled',
+                message: 'Async provisioning is temporarily disabled for this organization; retry without async: true',
+              },
+              503,
+            );
+          }
           // LLMO-7418 external-review Finding 9: see createMarket's async branch (and the
           // wasPending branch above) for the full rationale.
           await guardAgainstConcurrentProvisioning(
@@ -1905,6 +1942,17 @@ function SerenityController(context, log, env) {
       // in-request settle-poll + project-create/publish sequence), slated for removal once every
       // known caller has migrated to `async: true`.
       if (validateAsync(body)) {
+        // LLMO-7418 external-review Finding 15: server-side kill switch — see createMarket's
+        // async branch for the full rationale.
+        if (await isAsyncProvisioningKillSwitched(ctx, ctx?.params?.spaceCatId, log)) {
+          return createResponse(
+            {
+              error: 'asyncProvisioningDisabled',
+              message: 'Async provisioning is temporarily disabled for this organization; retry without async: true',
+            },
+            503,
+          );
+        }
         // LLMO-7418 external-review Finding 9: see the createMarket async branch above for the
         // full rationale — reconcile a stale in-flight attempt before minting a new one, since
         // beginProvisioningAttempt's own CAS has no staleness awareness.

src/support/serenity/serenity-active.js

@@ -47,6 +47,17 @@ export const SERENITY_FEATURE_FLAG_NAME = 'serenity';
  */
 export const SERENITY_UI_FEATURE_FLAG_NAME = 'serenity_ui';
 
+/**
+ * Server-side kill switch for the opt-in `async: true` provisioning path (LLMO-7418
+ * external-review Finding 15). Unlike `SERENITY_FEATURE_FLAG_NAME` above (an opt-IN
+ * rollout switch, off by default), this is an opt-OUT switch: the async path stays
+ * available by default, and an explicit `true` row disables it org-wide. Lets ops
+ * flip async provisioning off for one organization without a deploy, if it
+ * misbehaves in production — there was previously no way to do this short of a
+ * code change reverting every `async: true` call site at once.
+ */
+export const ASYNC_PROVISIONING_KILL_SWITCH_FLAG_NAME = 'serenity_async_provisioning_disabled';
+
 /**
  * Module-scoped TTL+size-bounded cache, mirroring the workspace-resolver cache
  * (warm Lambda containers reuse module state, so a Map here amortises the
@@ -239,3 +250,34 @@ export async function isSerenityUiActiveForOrg(ctx, spaceCatId, log) {
   const scopes = await readCachedFlagScopes(ctx, spaceCatId, SERENITY_UI_FEATURE_FLAG_NAME, log);
   return scopes?.orgRow?.flag_value === true;
 }
+
+/**
+ * LLMO-7418 external-review Finding 15: server-side kill switch for the opt-in
+ * `async: true` provisioning path. Reads the org-wide
+ * `LLMO/serenity_async_provisioning_disabled` feature flag (cached, same
+ * machinery as every other flag in this file).
+ *
+ * Deliberately reuses the SAME "absent/unreadable resolves to `false`"
+ * fail-safe shape as every other predicate here — it happens to be the safe
+ * default in both directions: for a rollout flag, `false` means "stay off";
+ * for this kill switch, `false` means "stay on" (async provisioning
+ * available). A transient PostgREST read failure must never silently disable
+ * async provisioning org-wide, so this is NOT inverted to fail closed.
+ *
+ * @param {object} ctx - Request context (uses
+ *   `ctx.dataAccess.services.postgrestClient`).
+ * @param {string} spaceCatId - SpaceCat organization UUID.
+ * @param {object} [log] - Optional logger (used to surface a missing client /
+ *   a read error without throwing on this hot path).
+ * @returns {Promise<boolean>} `true` only when the flag is explicitly on
+ *   (async provisioning is disabled for this organization).
+ */
+export async function isAsyncProvisioningKillSwitched(ctx, spaceCatId, log) {
+  const scopes = await readCachedFlagScopes(
+    ctx,
+    spaceCatId,
+    ASYNC_PROVISIONING_KILL_SWITCH_FLAG_NAME,
+    log,
+  );
+  return scopes?.orgRow?.flag_value === true;
+}

test/controllers/brands.test.js

@@ -5319,6 +5319,9 @@ describe('Brands Controller', () => {
         createAndEnqueueJob = sinon.stub().resolves({ getId: () => 'job-abc' }),
         promoteProvisioningFailed = sinon.stub().resolves(true),
         updateProvisioningJobId = sinon.stub().resolves(true),
+        // LLMO-7418 external-review Finding 15: kill switch off by default (async available);
+        // specific tests override it to resolve(true) to exercise the 503 gate.
+        isAsyncProvisioningKillSwitched = sinon.stub().resolves(false),
       } = {}) {
         const Mocked = await esmock('../../src/controllers/brands.js', {
           '../../src/support/serenity/brand-provisioning.js': {
@@ -5328,7 +5331,10 @@ describe('Brands Controller', () => {
             provisionBrandSubworkspaceBare:
               provisionBrandSubworkspaceBare || sinon.stub().resolves({ semrushSubWorkspaceId: 'ws-bare' }),
           },
-          '../../src/support/serenity/serenity-active.js': { isSerenityActiveForOrg },
+          '../../src/support/serenity/serenity-active.js': {
+            isSerenityActiveForOrg,
+            isAsyncProvisioningKillSwitched,
+          },
           '../../src/support/serenity/workspace-resolver.js': { resolveWorkspaceId },
           '../../src/support/serenity/async-job-runner.js': { createAndEnqueueJob },
           '../../src/support/serenity/handlers/provision-workspace-job.js': {
@@ -5519,6 +5525,28 @@ describe('Brands Controller', () => {
         expect(enqueueStub).to.not.have.been.called;
       });
 
+      it('returns 503 without enqueuing when the async kill switch is on (LLMO-7418 external-review Finding 15)', async () => {
+        const beginStub = sinon.stub().resolves(true);
+        const enqueueStub = sinon.stub().resolves({ getId: () => 'job-xyz' });
+        const controller = await buildController({
+          beginProvisioningAttempt: beginStub,
+          createAndEnqueueJob: enqueueStub,
+          isAsyncProvisioningKillSwitched: sinon.stub().resolves(true),
+        });
+
+        const response = await controller.createBrandForOrg({
+          ...context,
+          params: { spaceCatId: ORGANIZATION_ID },
+          data: { ...semrushData },
+          dataAccess: mockDataAccess,
+          attributes: { authInfo: { getType: () => 'ims', profile: { email: 'user@test.com' } } },
+        });
+
+        expect(response.status).to.equal(503);
+        expect(beginStub).to.not.have.been.called;
+        expect(enqueueStub).to.not.have.been.called;
+      });
+
       it('returns 400 without persisting a row when the organization has no Semrush parent workspace configured', async () => {
         const upsertStub = sinon.stub().resolves({ id: 'forced-id', name: 'New Brand' });
         const controller = await buildController({
@@ -5863,6 +5891,28 @@ describe('Brands Controller', () => {
         expect(enqueueStub.called).to.equal(false);
       });
 
+      it('Phase 4: bare create returns 503 without enqueuing when the async kill switch is on (LLMO-7418 external-review Finding 15)', async () => {
+        const beginStub = sinon.stub().resolves(true);
+        const enqueueStub = sinon.stub().resolves({ getId: () => 'job-xyz' });
+        const controller = await buildController({
+          beginProvisioningAttempt: beginStub,
+          createAndEnqueueJob: enqueueStub,
+          isAsyncProvisioningKillSwitched: sinon.stub().resolves(true),
+        });
+
+        const response = await controller.createBrandForOrg({
+          ...context,
+          params: { spaceCatId: ORGANIZATION_ID },
+          data: { name: 'New Brand', baseSiteId: 'site-123', async: true },
+          dataAccess: mockDataAccess,
+          attributes: { authInfo: { getType: () => 'ims', profile: { email: 'user@test.com' } } },
+        });
+
+        expect(response.status).to.equal(503);
+        expect(beginStub.called).to.equal(false);
+        expect(enqueueStub.called).to.equal(false);
+      });
+
       it('Phase 4: bare create returns 400 without persisting a row when the organization has no Semrush parent workspace configured', async () => {
         const upsertStub = sinon.stub().resolves({ id: 'forced-id', name: 'New Brand' });
         const controller = await buildController({

test/controllers/serenity.test.js

@@ -199,6 +199,7 @@ describe('SerenityController', () => {
   let resolveWorkspaceIdStub;
   let resolveBrandWorkspaceStub;
   let isSerenityActiveStub;
+  let isAsyncProvisioningKillSwitchedStub;
   let createTransportStub;
   let resolveBrandUuidStub;
   let getBrandAliasesStub;
@@ -237,6 +238,9 @@ describe('SerenityController', () => {
     // existing assertion that drives a brand-level route reaches its handler.
     // The "serenity inactive" describe overrides this to false.
     isSerenityActiveStub = sinon.stub().resolves(true);
+    // LLMO-7418 external-review Finding 15: kill switch off by default (async provisioning
+    // available); specific tests override it to resolve(true) to exercise the 503 gate.
+    isAsyncProvisioningKillSwitchedStub = sinon.stub().resolves(false);
     decommissionStub = sinon.stub().resolves();
     ensureSubworkspaceStub = sinon.stub().resolves(SUBWS);
     clearBrandWorkspaceCacheStub = sinon.stub();
@@ -352,6 +356,7 @@ describe('SerenityController', () => {
       },
       '../../src/support/serenity/serenity-active.js': {
         isSerenityActiveForBrand: isSerenityActiveStub,
+        isAsyncProvisioningKillSwitched: isAsyncProvisioningKillSwitchedStub,
       },
       '../../src/support/access-control-util.js': MockAccessControlUtil,
       '../../src/support/prompts-storage.js': {
@@ -1711,6 +1716,20 @@ describe('SerenityController', () => {
       expect(createAndEnqueueJobStub).to.not.have.been.called;
     });
 
+    it('createMarket answers 503 without enqueuing anything when the async kill switch is on (LLMO-7418 external-review Finding 15)', async () => {
+      isAsyncProvisioningKillSwitchedStub.resolves(true);
+      const controller = SerenityController({ env: {} }, fakeLog(), {});
+      const response = await controller.createMarket(fakeContext({
+        data: {
+          market: 'us', languageCode: 'en', brandDomain: 'x.com', brandNames: ['X'], async: true,
+        },
+      }));
+      expect(response.status).to.equal(503);
+      expect(guardAgainstConcurrentProvisioningStub).to.not.have.been.called;
+      expect(beginProvisioningAttemptStub).to.not.have.been.called;
+      expect(createAndEnqueueJobStub).to.not.have.been.called;
+    });
+
     it('createMarket runs the SAME synchronous orchestration it always has when async is absent (regression: default behavior unchanged)', async () => {
       orchestrateCreateMarketSubworkspaceStub.resolves({
         status: 201, body: { brandId: BRAND, geoTargetId: 2840, languageCode: 'en' },
@@ -2262,6 +2281,21 @@ describe('SerenityController', () => {
       expect(createAndEnqueueJobStub).to.not.have.been.called;
     });
 
+    it('activate answers 503 without enqueuing when async: true and the async kill switch is on (LLMO-7418 external-review Finding 15)', async () => {
+      isAsyncProvisioningKillSwitchedStub.resolves(true);
+      const brand = makeBrandModel({ getStatus: () => 'active' });
+      const controller = SerenityController({ env: {} }, fakeLog(), {});
+      const response = await controller.activate(fakeContext({
+        brand,
+        data: {
+          brandDomain: 'x.com', brandNames: ['X'], markets: [{ market: 'us', languageCode: 'en' }], async: true,
+        },
+      }));
+      expect(response.status).to.equal(503);
+      expect(beginProvisioningAttemptStub).to.not.have.been.called;
+      expect(createAndEnqueueJobStub).to.not.have.been.called;
+    });
+
     it('activate 400s when async is present but not a boolean', async () => {
       const brand = makeBrandModel({ getStatus: () => 'active' });
       const controller = SerenityController({ env: {} }, fakeLog(), {});
@@ -2462,6 +2496,19 @@ describe('SerenityController', () => {
       expect(createAndEnqueueJobStub).to.not.have.been.called;
     });
 
+    it('Phase 4: pending→active activation answers 503 without enqueuing when the async kill switch is on (LLMO-7418 external-review Finding 15)', async () => {
+      getBrandBaseSiteIdStub.resolves('primary-site');
+      isAsyncProvisioningKillSwitchedStub.resolves(true);
+      const brand = makeBrandModel({});
+      const controller = SerenityController({ env: {} }, fakeLog(), {});
+      const response = await controller.activate(fakeContext({
+        brand, data: { brandNames: ['X'], async: true },
+      }));
+      expect(response.status).to.equal(503);
+      expect(beginProvisioningAttemptStub).to.not.have.been.called;
+      expect(createAndEnqueueJobStub).to.not.have.been.called;
+    });
+
     it('Phase 4: pending→active activation 400s when async is present but not a boolean', async () => {
       getBrandBaseSiteIdStub.resolves('primary-site');
       const brand = makeBrandModel({});
@@ -2529,6 +2576,18 @@ describe('SerenityController', () => {
       expect(createAndEnqueueJobStub).to.not.have.been.called;
     });
 
+    it('Phase 4: bare reactivation answers 503 without enqueuing when the async kill switch is on (LLMO-7418 external-review Finding 15)', async () => {
+      isAsyncProvisioningKillSwitchedStub.resolves(true);
+      const brand = makeBrandModel({ getStatus: () => 'active' });
+      const controller = SerenityController({ env: {} }, fakeLog(), {});
+      const response = await controller.activate(fakeContext({
+        brand, data: { brandNames: ['X'], async: true },
+      }));
+      expect(response.status).to.equal(503);
+      expect(beginProvisioningAttemptStub).to.not.have.been.called;
+      expect(createAndEnqueueJobStub).to.not.have.been.called;
+    });
+
     it('Phase 4: bare reactivation 400s when async is present but not a boolean', async () => {
       const brand = makeBrandModel({ getStatus: () => 'active' });
       const controller = SerenityController({ env: {} }, fakeLog(), {});