fix(serenity): retry transient upstream/network errors during a workspace status poll (LLMO-7418 external-review Finding 14)

Previously any error from transport.getWorkspaceStatus() — including a
transient Semrush 503 or a network blip — landed in the generic outer catch
and permanently failed the whole provisioning attempt with zero retries,
even though the worker already has a bounded self-requeue mechanism built
for exactly this "try again shortly" case.

Classify by error.status: SerenityTransportError with 429/500/502/503/504,
or any error with no status at all (a raw network-level failure), is now
treated as transient — the poll result falls through as if the workspace
were merely "not ready", routing through the EXISTING self-requeue ladder
(same backoff/depth cap, no new mechanism). Everything else (a permanent
4xx like an expired IMS token) keeps today's fail-fast behavior.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: Iris Alexandrescu

src/support/serenity/handlers/provision-workspace-job.js

@@ -38,6 +38,23 @@ import {
  */
 export const PROVISION_WORKSPACE_JOB_TYPE = 'serenity-provision-workspace';
 
+// LLMO-7418 external-review Finding 14: retry classification for a status-poll failure.
+// `SerenityTransportError` always carries a numeric `.status` (the upstream HTTP status);
+// a raw network-level failure (fetch itself throwing — DNS, connection reset, timeout) has
+// none. Only these are treated as transient and routed through the existing bounded
+// self-requeue ladder below; everything else (a permanent 4xx like an expired/invalid IMS
+// token, or an unexpected non-transport error) keeps today's fail-fast behavior via the
+// outer catch.
+const RETRYABLE_TRANSPORT_STATUSES = new Set([429, 500, 502, 503, 504]);
+function isRetryableWorkspaceStatusError(error) {
+  const { status } = error ?? {};
+  if (typeof status !== 'number') {
+    // No upstream status at all — a network-level failure, not an application error.
+    return true;
+  }
+  return RETRYABLE_TRANSPORT_STATUSES.has(status);
+}
+
 /**
  * Hard cap on self-requeue depth. Live-verified settle time for a SUCCESSFUL create is
  * seconds, not minutes (LLMO-7352 incident data: ~10s); this ladder exists for the
@@ -382,7 +399,25 @@ export async function provisionWorkspaceHandler(context, job, accessToken) {
       }
     }
 
-    const statusResult = await transport.getWorkspaceStatus(candidate.workspaceId);
+    let statusResult;
+    try {
+      statusResult = await transport.getWorkspaceStatus(candidate.workspaceId);
+    } catch (error) {
+      if (!isRetryableWorkspaceStatusError(error)) {
+        throw error;
+      }
+      // Transient upstream/network failure — leave `statusResult` undefined so `status`
+      // below is `undefined`, which is neither ready nor terminal-failure, and this hop
+      // falls straight into the existing "still settling" self-requeue branch below (same
+      // bounded backoff/depth cap already used for an actual `not ready` poll result).
+      log?.warn?.('provision-workspace-job: transient error polling workspace status; treating as not-ready and self-requeuing', {
+        brandId,
+        attemptId,
+        semrushWorkspaceId: candidate.workspaceId,
+        error: error?.message,
+        status: error?.status,
+      });
+    }
     const status = statusResult?.status;
 
     if (isWorkspaceReady(status)) {

test/support/serenity/handlers/provision-workspace-job.test.js

@@ -757,6 +757,103 @@ describe('handlers/provision-workspace-job.js (LLMO-7352 / LLMO-7418)', () => {
     });
   });
 
+  describe('poll result: transient error retry classification (LLMO-7418 external-review Finding 14)', () => {
+    it('treats a 503 SerenityTransportError as transient and self-requeues instead of failing the attempt', async () => {
+      const err = new Error('Semrush GET .../status failed: 503');
+      err.status = 503;
+      transport.getWorkspaceStatus.rejects(err);
+      const { provisionWorkspaceHandler } = await loadHandler();
+      const job = makeJob(makeMetadata({ requeueDepth: 0 }));
+
+      const result = await provisionWorkspaceHandler(context, job, 'token');
+
+      expect(createAndEnqueueJobStub).to.have.been.calledOnce;
+      expect(promoteProvisioningFailedStub).to.not.have.been.called;
+      expect(result).to.deep.equal({ requeuedJobId: 'job-followup' });
+    });
+
+    it('retries 429/500/502/504 the same way', async () => {
+      for (const status of [429, 500, 502, 504]) {
+        const err = new Error(`upstream ${status}`);
+        err.status = status;
+        transport.getWorkspaceStatus.reset();
+        transport.getWorkspaceStatus.rejects(err);
+        createAndEnqueueJobStub.resetHistory();
+        promoteProvisioningFailedStub.resetHistory();
+        // eslint-disable-next-line no-await-in-loop
+        const { provisionWorkspaceHandler } = await loadHandler();
+        const job = makeJob(makeMetadata({ requeueDepth: 0 }));
+
+        // eslint-disable-next-line no-await-in-loop
+        const result = await provisionWorkspaceHandler(context, job, 'token');
+
+        expect(createAndEnqueueJobStub, `status ${status}`).to.have.been.calledOnce;
+        expect(promoteProvisioningFailedStub, `status ${status}`).to.not.have.been.called;
+        expect(result, `status ${status}`).to.deep.equal({ requeuedJobId: 'job-followup' });
+      }
+    });
+
+    it('treats a raw network failure (no .status) as transient and self-requeues', async () => {
+      transport.getWorkspaceStatus.rejects(new TypeError('fetch failed'));
+      const { provisionWorkspaceHandler } = await loadHandler();
+      const job = makeJob(makeMetadata({ requeueDepth: 0 }));
+
+      const result = await provisionWorkspaceHandler(context, job, 'token');
+
+      expect(createAndEnqueueJobStub).to.have.been.calledOnce;
+      expect(result).to.deep.equal({ requeuedJobId: 'job-followup' });
+    });
+
+    it('still fails fast on a permanent 401 (expired/invalid IMS token), never self-requeuing it', async () => {
+      const err = new Error('Semrush GET .../status failed: 401');
+      err.status = 401;
+      transport.getWorkspaceStatus.rejects(err);
+      const { provisionWorkspaceHandler } = await loadHandler();
+      const job = makeJob(makeMetadata());
+
+      await expect(provisionWorkspaceHandler(context, job, 'token')).to.be.rejectedWith('401');
+
+      expect(createAndEnqueueJobStub).to.not.have.been.called;
+      expect(promoteProvisioningFailedStub).to.have.been.calledOnceWith({
+        brandId: BRAND_ID,
+        attemptId: ATTEMPT_ID,
+        error: UNEXPECTED_ERROR_MESSAGE,
+        postgrestClient,
+      });
+    });
+
+    it('still fails fast on a permanent 400, never self-requeuing it', async () => {
+      const err = new Error('Semrush GET .../status failed: 400');
+      err.status = 400;
+      transport.getWorkspaceStatus.rejects(err);
+      const { provisionWorkspaceHandler } = await loadHandler();
+      const job = makeJob(makeMetadata());
+
+      await expect(provisionWorkspaceHandler(context, job, 'token')).to.be.rejectedWith('400');
+
+      expect(createAndEnqueueJobStub).to.not.have.been.called;
+    });
+
+    it('still respects the requeue depth cap for a transient error at the cap (fails, does not loop forever)', async () => {
+      const err = new Error('upstream 503');
+      err.status = 503;
+      transport.getWorkspaceStatus.rejects(err);
+      const { provisionWorkspaceHandler } = await loadHandler();
+      const job = makeJob(makeMetadata({ requeueDepth: MAX_PROVISION_REQUEUE_DEPTH }));
+
+      const result = await provisionWorkspaceHandler(context, job, 'token');
+
+      expect(createAndEnqueueJobStub).to.not.have.been.called;
+      expect(promoteProvisioningFailedStub).to.have.been.calledOnceWith({
+        brandId: BRAND_ID,
+        attemptId: ATTEMPT_ID,
+        error: REQUEUE_EXHAUSTED_MESSAGE,
+        postgrestClient,
+      });
+      expect(result).to.deep.equal({ provisioningStatus: 'failed' });
+    });
+  });
+
   describe('unexpected errors reaching the outer catch clean up an owned candidate (LLMO-7418 external-review Finding 16)', () => {
     it('cleans up a freshly-created candidate when an unexpected error is thrown BEFORE any self-requeue is enqueued', async () => {
       getBrandProvisioningStateStub.rejects(new Error('db read blip'));