Skip to content

Commit db1188a

Browse files
authored
feat: emr instance fleet priority allocation (#35731)
### Issue # (if applicable) Closes #35710 ### Reason for this change The CDK's [OnDemandAllocationStrategy](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_stepfunctions_tasks.EmrCreateCluster.OnDemandAllocationStrategy.html) enum only includes `LOWEST_PRICE`, but AWS EMR supports a prioritized allocation strategy that allows users to specify instance type priorities. This enhancement aligns the CDK with the full EMR API capabilities. ### Description of changes 1. Add `PRIORITIZED = 'prioritized'` to the `OnDemandAllocationStrategy` enum 2. Add an optional `priority?: number` field to `InstanceTypeConfigProperty` 3. Update the [InstanceTypeConfigPropertyToJson function](https://github.com/aws/aws-cdk/blob/3ec6d06c7c58e4f14b3fb114d7c35dc6d01794d9/packages/aws-cdk-lib/aws-stepfunctions-tasks/lib/emr/private/cluster-utils.ts#L91) to map the priority field to the CloudFormation output 4. **Added comprehensive unit tests** covering enum values, priority field usage, and CloudFormation template generation 5. **Added integration test** demonstrating real-world usage with multiple instance types and priorities ### Describe any new or updated permissions being added No new IAM permissions are required. This change only extends existing EMR cluster creation functionality with additional configuration options. ### Description of how you validated changes • **Unit tests**: Added 3 tests covering enum values, priority field functionality, and CloudFormation output validation • **Integration test**: Created end-to-end test with realistic EMR cluster configuration using prioritized allocation strategy • **Manual validation**: Verified CloudFormation templates generate correctly with AllocationStrategy: "prioritized" and Priority: N fields • **Backward compatibility**: Confirmed existing tests continue to pass without modification ### Checklist - [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 7956537 commit db1188a

13 files changed

Lines changed: 1579 additions & 0 deletions

packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/emr/integ.emr-create-cluster-with-prioritized-instance-fleet.js.snapshot/EmrCreateClusterPrioritizedTestDefaultTestDeployAssert7E92B105.assets.json

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/emr/integ.emr-create-cluster-with-prioritized-instance-fleet.js.snapshot/EmrCreateClusterPrioritizedTestDefaultTestDeployAssert7E92B105.template.json

Lines changed: 36 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/emr/integ.emr-create-cluster-with-prioritized-instance-fleet.js.snapshot/aws-cdk-emr-create-cluster-with-prioritized-instance-fleet.assets.json

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
{
2+
"Resources": {
3+
"EmrCreateClusterServiceRole5251910D": {
4+
"Type": "AWS::IAM::Role",
5+
"Properties": {
6+
"AssumeRolePolicyDocument": {
7+
"Statement": [
8+
{
9+
"Action": "sts:AssumeRole",
10+
"Effect": "Allow",
11+
"Principal": {
12+
"Service": "elasticmapreduce.amazonaws.com"
13+
}
14+
}
15+
],
16+
"Version": "2012-10-17"
17+
},
18+
"ManagedPolicyArns": [
19+
{
20+
"Fn::Join": [
21+
"",
22+
[
23+
"arn:",
24+
{
25+
"Ref": "AWS::Partition"
26+
},
27+
":iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"
28+
]
29+
]
30+
}
31+
]
32+
}
33+
},
34+
"EmrCreateClusterServiceRoleDefaultPolicyA8B4FA32": {
35+
"Type": "AWS::IAM::Policy",
36+
"Properties": {
37+
"PolicyDocument": {
38+
"Statement": [
39+
{
40+
"Action": "iam:PassRole",
41+
"Effect": "Allow",
42+
"Resource": {
43+
"Fn::GetAtt": [
44+
"EmrCreateClusterInstanceRoleC80466F5",
45+
"Arn"
46+
]
47+
}
48+
}
49+
],
50+
"Version": "2012-10-17"
51+
},
52+
"PolicyName": "EmrCreateClusterServiceRoleDefaultPolicyA8B4FA32",
53+
"Roles": [
54+
{
55+
"Ref": "EmrCreateClusterServiceRole5251910D"
56+
}
57+
]
58+
}
59+
},
60+
"EmrCreateClusterInstanceRoleC80466F5": {
61+
"Type": "AWS::IAM::Role",
62+
"Properties": {
63+
"AssumeRolePolicyDocument": {
64+
"Statement": [
65+
{
66+
"Action": "sts:AssumeRole",
67+
"Effect": "Allow",
68+
"Principal": {
69+
"Service": "ec2.amazonaws.com"
70+
}
71+
}
72+
],
73+
"Version": "2012-10-17"
74+
}
75+
}
76+
},
77+
"EmrCreateClusterInstanceProfileC1729180": {
78+
"Type": "AWS::IAM::InstanceProfile",
79+
"Properties": {
80+
"InstanceProfileName": {
81+
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
82+
},
83+
"Roles": [
84+
{
85+
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
86+
}
87+
]
88+
}
89+
},
90+
"SMRole49C19C48": {
91+
"Type": "AWS::IAM::Role",
92+
"Properties": {
93+
"AssumeRolePolicyDocument": {
94+
"Statement": [
95+
{
96+
"Action": "sts:AssumeRole",
97+
"Effect": "Allow",
98+
"Principal": {
99+
"Service": "states.amazonaws.com"
100+
}
101+
}
102+
],
103+
"Version": "2012-10-17"
104+
}
105+
}
106+
},
107+
"SMRoleDefaultPolicy34CA15C7": {
108+
"Type": "AWS::IAM::Policy",
109+
"Properties": {
110+
"PolicyDocument": {
111+
"Statement": [
112+
{
113+
"Action": [
114+
"elasticmapreduce:AddTags",
115+
"elasticmapreduce:DescribeCluster",
116+
"elasticmapreduce:RunJobFlow",
117+
"elasticmapreduce:TerminateJobFlows"
118+
],
119+
"Effect": "Allow",
120+
"Resource": "*"
121+
},
122+
{
123+
"Action": "iam:PassRole",
124+
"Effect": "Allow",
125+
"Resource": [
126+
{
127+
"Fn::GetAtt": [
128+
"EmrCreateClusterInstanceRoleC80466F5",
129+
"Arn"
130+
]
131+
},
132+
{
133+
"Fn::GetAtt": [
134+
"EmrCreateClusterServiceRole5251910D",
135+
"Arn"
136+
]
137+
}
138+
]
139+
},
140+
{
141+
"Action": "iam:CreateServiceLinkedRole",
142+
"Condition": {
143+
"StringEquals": {
144+
"iam:AWSServiceName": [
145+
"elasticmapreduce.amazonaws.com",
146+
"elasticmapreduce.amazonaws.com.cn"
147+
]
148+
}
149+
},
150+
"Effect": "Allow",
151+
"Resource": {
152+
"Fn::Join": [
153+
"",
154+
[
155+
"arn:",
156+
{
157+
"Ref": "AWS::Partition"
158+
},
159+
":iam::",
160+
{
161+
"Ref": "AWS::AccountId"
162+
},
163+
":role/aws-service-role/elasticmapreduce.amazonaws.com*/AWSServiceRoleForEMRCleanup*"
164+
]
165+
]
166+
},
167+
"Sid": "ElasticMapReduceServiceLinkedRole"
168+
},
169+
{
170+
"Action": [
171+
"events:DescribeRule",
172+
"events:PutRule",
173+
"events:PutTargets"
174+
],
175+
"Effect": "Allow",
176+
"Resource": {
177+
"Fn::Join": [
178+
"",
179+
[
180+
"arn:",
181+
{
182+
"Ref": "AWS::Partition"
183+
},
184+
":events:",
185+
{
186+
"Ref": "AWS::Region"
187+
},
188+
":",
189+
{
190+
"Ref": "AWS::AccountId"
191+
},
192+
":rule/StepFunctionsGetEventForEMRRunJobFlowRule"
193+
]
194+
]
195+
}
196+
}
197+
],
198+
"Version": "2012-10-17"
199+
},
200+
"PolicyName": "SMRoleDefaultPolicy34CA15C7",
201+
"Roles": [
202+
{
203+
"Ref": "SMRole49C19C48"
204+
}
205+
]
206+
}
207+
},
208+
"SM934E715A": {
209+
"Type": "AWS::StepFunctions::StateMachine",
210+
"Properties": {
211+
"DefinitionString": {
212+
"Fn::Join": [
213+
"",
214+
[
215+
"{\"StartAt\":\"EmrCreateCluster\",\"States\":{\"EmrCreateCluster\":{\"End\":true,\"Type\":\"Task\",\"Resource\":\"arn:",
216+
{
217+
"Ref": "AWS::Partition"
218+
},
219+
":states:::elasticmapreduce:createCluster.sync\",\"Parameters\":{\"Instances\":{\"InstanceFleets\":[{\"InstanceFleetType\":\"CORE\",\"InstanceTypeConfigs\":[{\"InstanceType\":\"m5.large\",\"Priority\":0,\"WeightedCapacity\":1},{\"InstanceType\":\"m5.xlarge\",\"Priority\":1,\"WeightedCapacity\":2}],\"LaunchSpecifications\":{\"OnDemandSpecification\":{\"AllocationStrategy\":\"prioritized\"}},\"Name\":\"Core\",\"TargetOnDemandCapacity\":2}],\"KeepJobFlowAliveWhenNoSteps\":true},\"JobFlowRole\":\"",
220+
{
221+
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
222+
},
223+
"\",\"Name\":\"PrioritizedCluster\",\"ServiceRole\":\"",
224+
{
225+
"Ref": "EmrCreateClusterServiceRole5251910D"
226+
},
227+
"\",\"ReleaseLabel\":\"emr-5.36.1\",\"Tags\":[{\"Key\":\"for-use-with-amazon-emr-managed-policies\",\"Value\":\"true\"}],\"VisibleToAllUsers\":true}}}}"
228+
]
229+
]
230+
},
231+
"RoleArn": {
232+
"Fn::GetAtt": [
233+
"SMRole49C19C48",
234+
"Arn"
235+
]
236+
}
237+
},
238+
"DependsOn": [
239+
"SMRoleDefaultPolicy34CA15C7",
240+
"SMRole49C19C48"
241+
],
242+
"UpdateReplacePolicy": "Delete",
243+
"DeletionPolicy": "Delete"
244+
}
245+
},
246+
"Parameters": {
247+
"BootstrapVersion": {
248+
"Type": "AWS::SSM::Parameter::Value<String>",
249+
"Default": "/cdk-bootstrap/hnb659fds/version",
250+
"Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
251+
}
252+
},
253+
"Rules": {
254+
"CheckBootstrapVersion": {
255+
"Assertions": [
256+
{
257+
"Assert": {
258+
"Fn::Not": [
259+
{
260+
"Fn::Contains": [
261+
[
262+
"1",
263+
"2",
264+
"3",
265+
"4",
266+
"5"
267+
],
268+
{
269+
"Ref": "BootstrapVersion"
270+
}
271+
]
272+
}
273+
]
274+
},
275+
"AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
276+
}
277+
]
278+
}
279+
}
280+
}

packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/emr/integ.emr-create-cluster-with-prioritized-instance-fleet.js.snapshot/cdk.out

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/emr/integ.emr-create-cluster-with-prioritized-instance-fleet.js.snapshot/integ.json

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)