feat(serenity): async Semrush-market AI-prompt provisioning (#3194)

.env.example

@@ -74,6 +74,25 @@ SLACK_URL_WORKSPACE_EXTERNAL=https://dummy-slack.example.com
 # ── Queue URLs & S3 buckets (dummy values, not called locally) ────────────────
 AUDIT_JOBS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/000000000000/dummy-audits
 REPORT_JOBS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/000000000000/dummy-reports
+# Async Semrush-market prompt generation (#3194). Queue for the serenity-job-runner
+# worker; the DRS generation Lambda name/ARN it synchronously invokes (cross-account
+# wiring: spacecat-infrastructure#780); and the rollout flag that turns the three
+# onboarding entry points from synchronous verbatim-catalogue tracking to async
+# DRS-backed generation. Flag default-off — unset means unchanged behavior.
+SERENITY_JOB_RUNNER_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/000000000000/dummy-serenity-jobs
+# Dedicated market-generation queue (separate from the shared runner queue above);
+# its DLQ is deliberately not auto-redriven — recovery is the lease-aware runbook.
+SERENITY_MARKET_JOBS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/000000000000/dummy-serenity-market-jobs
+# MUST be the DRS Lambda's FULL cross-account ARN, never a bare function name: the
+# worker invokes with a region-only client and no cross-account creds (cross-account
+# invoke works via the IAM grant + DRS resource policy), so a bare name resolves in the
+# CALLER account (spacecat) and fails ResourceNotFound. Name pattern
+# drs-v2-PromptGenerationSemrushMarket-<env>; DRS accounts: dev 489975610310,
+# prod 458960552570. This ARN MUST equal infra#780's drs_generation_lambda_arn.
+DRS_PROMPT_GENERATION_FUNCTION=arn:aws:lambda:us-east-1:489975610310:function:drs-v2-PromptGenerationSemrushMarket-dev
+# Generation model sent to DRS (canonical contract default; override per env).
+DRS_PROMPT_GENERATION_MODEL=gpt-5-nano
+SERENITY_ASYNC_PROMPT_GEN=false
 S3_REPORT_BUCKET=dummy-report-bucket
 S3_MYSTIQUE_BUCKET=dummy-mystique-bucket
 S3_CONFIG_BUCKET=dummy-config-bucket

docs/openapi/api.yaml

@@ -172,6 +172,10 @@ paths:
     $ref: './serenity-api.yaml#/v2-serenity-prompts-job-by-id'
   /v2/orgs/{spaceCatId}/brands/{brandId}/serenity/markets:
     $ref: './serenity-api.yaml#/v2-serenity-markets'
+  /v2/orgs/{spaceCatId}/brands/{brandId}/serenity/markets/generation/jobs/{jobId}:
+    $ref: './serenity-api.yaml#/v2-serenity-market-generation-job-by-id'
+  /v2/orgs/{spaceCatId}/brands/{brandId}/serenity/markets/generation/jobs/{jobId}/reauth:
+    $ref: './serenity-api.yaml#/v2-serenity-market-generation-job-reauth'
   /v2/orgs/{spaceCatId}/brands/{brandId}/serenity/markets/{geoTargetId}/{languageCode}:
     $ref: './serenity-api.yaml#/v2-serenity-market-by-slice'
   /v2/orgs/{spaceCatId}/brands/{brandId}/serenity/tags:

docs/openapi/schemas.yaml

@@ -12259,6 +12259,72 @@ SerenityPromptsJobStatus:
             retryable: { type: boolean }
         - { type: 'null' }
 
+SerenityMarketGenerationJobStatus:
+  type: object
+  description: |
+    Poll response for an async Semrush-market prompt-generation job
+    (serenity-docs#443). Secret-free by design — the job's internal metadata
+    (the delegated Semrush promise token) is never exposed.
+  required: [jobId, jobType, status, result, error]
+  properties:
+    jobId:
+      type: string
+      format: uuid
+    jobType:
+      type: string
+      enum: [generateSemrushMarket]
+    status:
+      type: string
+      enum: [IN_PROGRESS, COMPLETED, FAILED]
+    result:
+      description: The job result once COMPLETED; null otherwise.
+      oneOf:
+        - type: object
+          additionalProperties: false
+          properties:
+            promptCount: { type: integer }
+            projectId: { type: string }
+            published: { type: boolean }
+            verdict: { type: string, enum: [ship] }
+        - { type: 'null' }
+    error:
+      description: The sanitized public failure envelope once FAILED; null otherwise.
+      oneOf:
+        - type: object
+          additionalProperties: false
+          required: [code, message, retryable, needsReauth]
+          properties:
+            code: { type: string }
+            message: { type: string }
+            retryable: { type: boolean }
+            needsReauth:
+              type: boolean
+              description: True when the job stalled on an expired Semrush token; use the /reauth endpoint.
+        - { type: 'null' }
+    createdAt:
+      type: string
+      format: date-time
+      description: When the job was created.
+    updatedAt:
+      type: string
+      format: date-time
+      description: When the job record was last updated.
+
+SerenityMarketGenerationJobAccepted:
+  type: object
+  description: The 202 acknowledgement returned when a generation job is (re-)enqueued.
+  required: [jobId, jobType, status]
+  properties:
+    jobId:
+      type: string
+      format: uuid
+    jobType:
+      type: string
+      enum: [generateSemrushMarket]
+    status:
+      type: string
+      enum: [IN_PROGRESS]
+
 SerenityUpdatePromptRequest:
   type: object
   description: |

docs/openapi/serenity-api.yaml

@@ -473,6 +473,125 @@ v2-serenity-prompts-job-by-id:
         description: Organization, brand, or job not found, serenity is not active for the brand, or the job belongs to a different brand.
       '500': { $ref: './responses.yaml#/500' }
 
+# ─── Async Semrush-market prompt generation (jobs) ────────────────────────────
+
+v2-serenity-market-generation-job-by-id:
+  parameters:
+    - name: spaceCatId
+      in: path
+      required: true
+      description: SpaceCat Organization ID (UUID)
+      schema:
+        type: string
+        format: uuid
+    - name: brandId
+      in: path
+      required: true
+      description: Brand ID (UUID-only on the /serenity/* surface)
+      schema:
+        type: string
+        format: uuid
+    - name: jobId
+      in: path
+      required: true
+      description: >-
+        Async job id returned as `promptGeneration.jobId` by a create-market /
+        activate / create-brand response when async prompt generation is enabled.
+      schema:
+        type: string
+        format: uuid
+  get:
+    tags: [serenity]
+    summary: Poll an async Semrush-market prompt-generation job
+    description: |
+      Returns the status and, on completion, the result of an async
+      `serenity-generate-semrush-market` job (serenity-docs#443). Onboarding a
+      market/site with "include Semrush prompts" generates real monitoring prompts
+      from the Semrush topic catalogue via a synchronous DRS call, then writes them
+      to the market. This endpoint is guarded by the shared job-scoping primitive
+      (jobType allowlist + site/brand ownership); a job that does not belong to the
+      addressed brand 404s exactly like a missing job. The response is secret-free —
+      only `{ jobId, jobType, status, result, error }`, and NEVER a promise token.
+
+      On `FAILED` with `error.needsReauth = true`, the caller's Semrush promise
+      token was rejected by IMS; re-authenticate via the `/reauth` sibling.
+    operationId: getSerenityMarketGenerationJobStatus
+    security:
+      - session_token: []
+    responses:
+      '200':
+        description: Job status (result when COMPLETED / error when FAILED).
+        content:
+          application/json:
+            schema: { $ref: './schemas.yaml#/SerenityMarketGenerationJobStatus' }
+      '400': { $ref: './responses.yaml#/400' }
+      '404':
+        description: Organization, brand, or job not found, serenity is not active for the brand, or the job belongs to a different brand.
+      '500': { $ref: './responses.yaml#/500' }
+
+v2-serenity-market-generation-job-reauth:
+  parameters:
+    - name: spaceCatId
+      in: path
+      required: true
+      description: SpaceCat Organization ID (UUID)
+      schema:
+        type: string
+        format: uuid
+    - name: brandId
+      in: path
+      required: true
+      description: Brand ID (UUID-only on the /serenity/* surface)
+      schema:
+        type: string
+        format: uuid
+    - name: jobId
+      in: path
+      required: true
+      description: The generation job id awaiting re-authentication.
+      schema:
+        type: string
+        format: uuid
+  post:
+    tags: [serenity]
+    summary: Re-authenticate a generation job whose Semrush token expired
+    description: |
+      Re-authorizes a `serenity-generate-semrush-market` job that failed with
+      `NEEDS_REAUTH` (its delegated Semrush promise token was rejected by IMS).
+      STRICT, fail-closed authorization: the caller must be the SAME IMS user that
+      originally enqueued the job (single stable `user_id` claim). A replacement
+      token is accepted ONLY on the explicit Semrush promise pair
+      (`x-promise-audience: semrush`); the job record is updated atomically and the
+      SAME job id is re-enqueued. Returns 202 with the job handle.
+    operationId: reauthSerenityMarketGenerationJob
+    security:
+      - session_token: []
+    parameters:
+      - name: x-promise-audience
+        in: header
+        required: true
+        description: Must be `semrush`.
+        schema: { type: string, enum: [semrush] }
+      - name: x-promise-token
+        in: header
+        required: false
+        description: The re-minted Semrush promise token (when the client forwards one).
+        schema: { type: string }
+    responses:
+      '202':
+        description: Re-authentication accepted; the job was re-enqueued.
+        content:
+          application/json:
+            schema: { $ref: './schemas.yaml#/SerenityMarketGenerationJobAccepted' }
+      '400': { $ref: './responses.yaml#/400' }
+      '403':
+        description: The caller is not the original requester (fail-closed identity check).
+      '404':
+        description: Organization, brand, or job not found, or the job belongs to a different brand.
+      '409':
+        description: The job is not awaiting re-authentication.
+      '500': { $ref: './responses.yaml#/500' }
+
 # ─── Markets ──────────────────────────────────────────────────────────────────
 
 v2-serenity-markets: