From 01c51fcbb6bed972bd6415ab987d17aca69ff4d3 Mon Sep 17 00:00:00 2001 From: Akol125 Date: Wed, 8 Oct 2025 13:27:46 +0100 Subject: [PATCH 1/6] create data quality s3 buckets --- terraform/s3_dq_reports.tf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 terraform/s3_dq_reports.tf diff --git a/terraform/s3_dq_reports.tf b/terraform/s3_dq_reports.tf new file mode 100644 index 0000000000..8f0eecd923 --- /dev/null +++ b/terraform/s3_dq_reports.tf @@ -0,0 +1,24 @@ +# Create s3 Bucket with conditional destroy for pr environments +resource "aws_s3_bucket" "data_quality_reports_bucket" { + bucket = "imms-${local.short_prefix}-data_quality_reports" + force_destroy = local.is_temp + +} + +# Block public access to the bucket +resource "aws_s3_bucket_public_access_block" "data_quality_reports_bucket_public_access_block" { + bucket = aws_s3_bucket.data_quality_reports_bucket.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + + +resource "aws_s3_bucket_versioning" "dq_source_versioning" { + bucket = aws_s3_bucket.data_quality_reports_bucket.bucket + versioning_configuration { + status = "Enabled" + } +} \ No newline at end of file From 60db91d33462c50d0f5ddd3c27113d2a5ab5b687 Mon Sep 17 00:00:00 2001 From: Akol125 Date: Wed, 8 Oct 2025 14:37:52 +0100 Subject: [PATCH 2/6] add s3 lifecycle and policies --- terraform/s3_dq_reports.tf | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/terraform/s3_dq_reports.tf b/terraform/s3_dq_reports.tf index 8f0eecd923..a07016d070 100644 --- a/terraform/s3_dq_reports.tf +++ b/terraform/s3_dq_reports.tf @@ -15,10 +15,45 @@ resource "aws_s3_bucket_public_access_block" "data_quality_reports_bucket_public restrict_public_buckets = true } +resource "aws_s3_bucket_lifecycle_configuration" "data_destinations" { + bucket = aws_s3_bucket.data_quality_reports_bucket.id + + rule { + id = "DeleteFilesFromForwardedFile" + status = "Enabled" + filter { + } + + expiration { + days = 14 + } + } +} + + +# Add versioning to prevent against accidental deletes resource "aws_s3_bucket_versioning" "dq_source_versioning" { bucket = aws_s3_bucket.data_quality_reports_bucket.bucket versioning_configuration { status = "Enabled" } +} + + +# If used should attached to lambda or any aws service that needs to perform any operation +resource "aws_iam_policy" "s3_dq_access" { + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["s3:GetObject", "s3:PutObject", "s3:ListBucket"] + Resource = [ + aws_s3_bucket.data_quality_reports_bucket.arn, + "${aws_s3_bucket.data_quality_reports_bucket.arn}/*" + ] + } + ] + }) } \ No newline at end of file From 18f1834d9abc24a4e0243b14c487b00ed443d53b Mon Sep 17 00:00:00 2001 From: Akol125 Date: Wed, 8 Oct 2025 14:40:25 +0100 Subject: [PATCH 3/6] gen validation error --- terraform/s3_dq_reports.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/s3_dq_reports.tf b/terraform/s3_dq_reports.tf index a07016d070..b2a4bfa21c 100644 --- a/terraform/s3_dq_reports.tf +++ b/terraform/s3_dq_reports.tf @@ -19,7 +19,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "data_destinations" { bucket = aws_s3_bucket.data_quality_reports_bucket.id rule { - id = "DeleteFilesFromForwardedFile" + id = "GenericValidationReports" status = "Enabled" filter { From 15f2fff1d45a8e6e8ebb9f0296e3173382005933 Mon Sep 17 00:00:00 2001 From: Akol125 Date: Wed, 8 Oct 2025 15:20:11 +0100 Subject: [PATCH 4/6] change lifecycle name --- terraform/s3_dq_reports.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/s3_dq_reports.tf b/terraform/s3_dq_reports.tf index b2a4bfa21c..82963afb02 100644 --- a/terraform/s3_dq_reports.tf +++ b/terraform/s3_dq_reports.tf @@ -15,7 +15,7 @@ resource "aws_s3_bucket_public_access_block" "data_quality_reports_bucket_public restrict_public_buckets = true } -resource "aws_s3_bucket_lifecycle_configuration" "data_destinations" { +resource "aws_s3_bucket_lifecycle_configuration" "data_quality_reports" { bucket = aws_s3_bucket.data_quality_reports_bucket.id rule { From 80c3384fbbce9e954a8f88293a8860939464f68a Mon Sep 17 00:00:00 2001 From: Akol125 Date: Wed, 8 Oct 2025 15:46:12 +0100 Subject: [PATCH 5/6] rename bucket --- terraform/s3_dq_reports.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/s3_dq_reports.tf b/terraform/s3_dq_reports.tf index 82963afb02..f619af32d6 100644 --- a/terraform/s3_dq_reports.tf +++ b/terraform/s3_dq_reports.tf @@ -1,6 +1,6 @@ # Create s3 Bucket with conditional destroy for pr environments resource "aws_s3_bucket" "data_quality_reports_bucket" { - bucket = "imms-${local.short_prefix}-data_quality_reports" + bucket = "${local.short_prefix}-data-quality-reports" force_destroy = local.is_temp } From b84a5775c739ac3d3b2e8488bb70d8cc2aa4ee46 Mon Sep 17 00:00:00 2001 From: Akol125 Date: Thu, 9 Oct 2025 13:49:27 +0100 Subject: [PATCH 6/6] VED-795: Add deny rules for DQ Report --- terraform/s3_dq_reports.tf | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/terraform/s3_dq_reports.tf b/terraform/s3_dq_reports.tf index f619af32d6..3a44abd785 100644 --- a/terraform/s3_dq_reports.tf +++ b/terraform/s3_dq_reports.tf @@ -1,6 +1,6 @@ # Create s3 Bucket with conditional destroy for pr environments resource "aws_s3_bucket" "data_quality_reports_bucket" { - bucket = "${local.short_prefix}-data-quality-reports" + bucket = "imms-${local.resource_scope}-data-quality-reports" force_destroy = local.is_temp } @@ -41,14 +41,14 @@ resource "aws_s3_bucket_versioning" "dq_source_versioning" { } -# If used should attached to lambda or any aws service that needs to perform any operation +# If used should attach to lambda or any aws service that needs to perform any operation resource "aws_iam_policy" "s3_dq_access" { policy = jsonencode({ Version = "2012-10-17" Statement = [ { Effect = "Allow" - Action = ["s3:GetObject", "s3:PutObject", "s3:ListBucket"] + Action = ["s3:PutObject"] Resource = [ aws_s3_bucket.data_quality_reports_bucket.arn, "${aws_s3_bucket.data_quality_reports_bucket.arn}/*" @@ -56,4 +56,33 @@ resource "aws_iam_policy" "s3_dq_access" { } ] }) +} + + +resource "aws_s3_bucket_policy" "data_quality_bucket_policy" { + bucket = aws_s3_bucket.data_quality_reports_bucket.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "data_quality_bucket_policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = { + AWS = "*" + } + Action = "s3:*" + Resource = [ + aws_s3_bucket.data_quality_reports_bucket.arn, + "${aws_s3_bucket.data_quality_reports_bucket.arn}/*" + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) } \ No newline at end of file