From 07dd48f0e58e2d53583d4b0b610e98ad8d3cf123 Mon Sep 17 00:00:00 2001 From: Anurag Mittal <1321012+anurag4DSB@users.noreply.github.com> Date: Tue, 8 Sep 2026 08:16:30 +0200 Subject: [PATCH 1/2] CLDSRV-992: Bound and settle the 1000-object writes in Multi-Object Delete setup The "Multi-Object Delete Success" beforeEach uploads 1000 objects behind what looks like a concurrency limiter of 20. It is not one. objects.map() invokes all 1000 async calls synchronously; the first 20 push and the other 980 all park on the same `await Promise.race(queued)`. When the first put settles they all wake, and each splices one entry off the head of the queue regardless of whether that entry had settled, so the queue empties, every waiter's `while` test goes false, and the remaining puts fire together. Running the limiter against an instrumented stub gives a peak of 999 concurrent puts against an intended 20. That is the ServiceUnavailable seen in run 33681270087: a thousand simultaneous puts, not twenty. It also explains why one failure becomes seven. Promise.all rejects on the first error while the other puts are still in flight, so the hook throws with writes outstanding, and the bucket cannot be deleted. All four suites in this file share the bucket name multi-object-delete-234-634, so once it is left behind with objects in it, every later suite's cleanup fails with BucketNotEmpty -- which is exactly the shape of the run: one ServiceUnavailable followed by six BucketNotEmpty. Two changes, both already present on development/9.3 and newer in one form or another: - Replace the limiter with a fixed pool of 20 workers draining a shared queue. Concurrency is genuinely bounded and every put is settled before the hook returns, on the failure path too, so teardown can no longer race them. - Empty the bucket before deleting it in all four cleanup hooks. The 9.3 line gained this in 9eb5777a8, but that commit is an aws-sdk v3 migration of the whole file and is not something to cherry-pick onto 9.2; this is the equivalent written for the v2 client, and nothing else from it. Clears CLDSRV-992 row F10 (all seven failures). Issue: CLDSRV-992 --- .../test/object/multiObjectDelete.js | 55 ++++++++++++------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js index 10de2996e1..085cc27e12 100644 --- a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js +++ b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js @@ -68,29 +68,37 @@ describe('Multi-Object Delete Success', function success() { objects.push(`${key}${i}`); } const parallel = 20; - const queued = []; - const putObjectWithLimit = async key => { - while (queued.length >= parallel) { - await Promise.race(queued); - queued.splice(0, queued.findIndex(p => p === queued[0]) + 1); + // Fixed pool draining a shared queue. The previous limiter peaked + // at 999 concurrent puts and abandoned the rest on first failure. + const pending = objects.slice(); + const putErrors = []; + await Promise.all(Array.from({ length: parallel }, async () => { + while (pending.length > 0) { + const objectKey = pending.shift(); + try { + await s3.putObject({ + Bucket: bucketName, + Key: objectKey, + Body: 'somebody', + }).promise(); + } catch (err) { + putErrors.push(err); + } } - const result = s3.putObject({ - Bucket: bucketName, - Key: key, - Body: 'somebody', - }).promise(); - queued.push(result); - return result; - }; - const putPromises = objects.map(key => putObjectWithLimit(key)); - await Promise.all(putPromises); + })); + if (putErrors.length > 0) { + throw putErrors[0]; + } } catch (err) { process.stdout.write(`Error creating objects: ${err}\n`); throw err; } }); - afterEach(() => s3.deleteBucket({ Bucket: bucketName }).promise()); + afterEach(async () => { + await bucketUtil.empty(bucketName); + await s3.deleteBucket({ Bucket: bucketName }).promise(); + }); it('should batch delete 1000 objects', done => { const objects = createObjectsList(1000); @@ -150,7 +158,10 @@ describe('Multi-Object Delete Error Responses', () => { }); }); - afterEach(() => s3.deleteBucket({ Bucket: bucketName }).promise()); + afterEach(async () => { + await bucketUtil.empty(bucketName); + await s3.deleteBucket({ Bucket: bucketName }).promise(); + }); it('should return error if request deletion of more than 1000 objects', () => { @@ -240,7 +251,10 @@ describe('Multi-Object Delete Access', function access() { }); }); - after(() => s3.deleteBucket({ Bucket: bucketName }).promise()); + after(async () => { + await bucketUtil.empty(bucketName); + await s3.deleteBucket({ Bucket: bucketName }).promise(); + }); it('should return access denied error for each object where no acl ' + 'permission', () => { @@ -339,7 +353,10 @@ describeSkipIfCeph('Multi-Object Delete with Object Lock', () => { }); }); - after(() => s3.deleteBucket({ Bucket: bucketName }).promise()); + after(async () => { + await bucketUtil.empty(bucketName); + await s3.deleteBucket({ Bucket: bucketName }).promise(); + }); it('should not delete locked objects', () => { const objects = createObjectsList(5, versionIds); From 28d0696636153d72f0de241521573f843c776b21 Mon Sep 17 00:00:00 2001 From: Anurag Mittal <1321012+anurag4DSB@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:58:50 +0200 Subject: [PATCH 2/2] tmp: run s3c-ft-tests-v1 10x for flakiness validation Stacked on the uploadPartCopy fixture fix so the stress signal is not polluted by row F4/F8. Dropped before review. Precedent: a3847d604. --- .github/workflows/tests.yaml | 44 ++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9cce05736c..7e1bcf7823 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -242,6 +242,7 @@ jobs: cache-to: type=gha,mode=max,scope=mongodb multiple-backend: + if: false # CLDSRV-992 stress: unrelated job runs-on: ubuntu-24.04 needs: build env: @@ -301,6 +302,7 @@ jobs: if: always() mongo-v0-ft-tests: + if: false # CLDSRV-992 stress: unrelated job runs-on: ubuntu-24.04 needs: build env: @@ -360,6 +362,7 @@ jobs: if: always() mongo-v1-ft-tests: + if: false # CLDSRV-992 stress: unrelated job runs-on: ubuntu-24.04 needs: build env: @@ -424,6 +427,7 @@ jobs: # All tests use non federation images file-ft-tests: + if: false # CLDSRV-992 stress: unrelated job strategy: matrix: include: @@ -494,17 +498,39 @@ jobs: # Configure and run as Integration run S3C tests s3c-ft-tests: strategy: + fail-fast: false matrix: include: - - vformat: v0 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-1 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-2 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-3 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-4 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-5 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-6 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-7 + - vformat: v1 + enable-null-compat: '' + job-name: s3c-ft-tests-v1-8 + - vformat: v1 enable-null-compat: '' - job-name: s3c-ft-tests-v0 - - vformat: v0 - enable-null-compat: 'true' - job-name: s3c-ft-tests-v0-null-compat + job-name: s3c-ft-tests-v1-9 - vformat: v1 enable-null-compat: '' - job-name: s3c-ft-tests-v1 + job-name: s3c-ft-tests-v1-10 name: ${{ matrix.job-name }} runs-on: ubuntu-24.04 needs: build @@ -624,7 +650,7 @@ jobs: utapi-v2-tests: runs-on: ubuntu-24.04 needs: build - if: always() + if: false # CLDSRV-992 stress: unrelated job env: ENABLE_UTAPI_V2: t S3BACKEND: mem @@ -668,6 +694,7 @@ jobs: if: always() sur-tests: + if: false # CLDSRV-992 stress: unrelated job runs-on: ubuntu-24.04 needs: build strategy: @@ -726,6 +753,7 @@ jobs: if: always() kmip-ft-tests: + if: false # CLDSRV-992 stress: unrelated job runs-on: ubuntu-24.04 needs: build env: @@ -786,6 +814,7 @@ jobs: if: always() kmip-cluster-ft-tests: + if: false # CLDSRV-992 stress: unrelated job runs-on: ubuntu-latest needs: build env: @@ -849,6 +878,7 @@ jobs: # This test with the final yarn run ft_sse_arn covers more code than the kmip tests sse-kms-migration-tests: + if: false # CLDSRV-992 stress: unrelated job strategy: fail-fast: false # prevent cancel if one 1 matrix option fails matrix: