Skip to content

Commit 3fe76c8

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: [google-cloud-dataproc] Add ProvisioningModelMix to support mixing of spot and standard instances for secondary workers (#13169)
- [ ] Regenerate this pull request now. feat: Add support for configuration of bootdisk IOPS and throughput when bootdisk is a hyperdisk PiperOrigin-RevId: 686008645 Source-Link: googleapis/googleapis@2f49d44 Source-Link: googleapis/googleapis-gen@5dc57b0 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRhdGFwcm9jLy5Pd2xCb3QueWFtbCIsImgiOiI1ZGM1N2IwMGQ5YTEzNTVkMDIwNDk2YWJlNTYwMGU3ODIwMmQ1ZDNiIn0= --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 20472b8 commit 3fe76c8

4 files changed

Lines changed: 118 additions & 0 deletions

File tree

‎packages/google-cloud-dataproc/google/cloud/dataproc_v1/types/clusters.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1110,6 +1110,10 @@ class InstanceFlexibilityPolicy(proto.Message):
11101110
and provisioning models.
11111111
11121112
Attributes:
1113+
provisioning_model_mix (google.cloud.dataproc_v1.types.InstanceFlexibilityPolicy.ProvisioningModelMix):
1114+
Optional. Defines how the Group selects the
1115+
provisioning model to ensure required
1116+
reliability.
11131117
instance_selection_list (MutableSequence[google.cloud.dataproc_v1.types.InstanceFlexibilityPolicy.InstanceSelection]):
11141118
Optional. List of instance selection options
11151119
that the group will use when creating new VMs.
@@ -1118,6 +1122,51 @@ class InstanceFlexibilityPolicy(proto.Message):
11181122
results in the group.
11191123
"""
11201124

1125+
class ProvisioningModelMix(proto.Message):
1126+
r"""Defines how Dataproc should create VMs with a mixture of
1127+
provisioning models.
1128+
1129+
1130+
.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields
1131+
1132+
Attributes:
1133+
standard_capacity_base (int):
1134+
Optional. The base capacity that will always use Standard
1135+
VMs to avoid risk of more preemption than the minimum
1136+
capacity you need. Dataproc will create only standard VMs
1137+
until it reaches standard_capacity_base, then it will start
1138+
using standard_capacity_percent_above_base to mix Spot with
1139+
Standard VMs. eg. If 15 instances are requested and
1140+
standard_capacity_base is 5, Dataproc will create 5 standard
1141+
VMs and then start mixing spot and standard VMs for
1142+
remaining 10 instances.
1143+
1144+
This field is a member of `oneof`_ ``_standard_capacity_base``.
1145+
standard_capacity_percent_above_base (int):
1146+
Optional. The percentage of target capacity that should use
1147+
Standard VM. The remaining percentage will use Spot VMs. The
1148+
percentage applies only to the capacity above
1149+
standard_capacity_base. eg. If 15 instances are requested
1150+
and standard_capacity_base is 5 and
1151+
standard_capacity_percent_above_base is 30, Dataproc will
1152+
create 5 standard VMs and then start mixing spot and
1153+
standard VMs for remaining 10 instances. The mix will be 30%
1154+
standard and 70% spot.
1155+
1156+
This field is a member of `oneof`_ ``_standard_capacity_percent_above_base``.
1157+
"""
1158+
1159+
standard_capacity_base: int = proto.Field(
1160+
proto.INT32,
1161+
number=1,
1162+
optional=True,
1163+
)
1164+
standard_capacity_percent_above_base: int = proto.Field(
1165+
proto.INT32,
1166+
number=2,
1167+
optional=True,
1168+
)
1169+
11211170
class InstanceSelection(proto.Message):
11221171
r"""Defines machines types and a rank to which the machines types
11231172
belong.
@@ -1175,6 +1224,11 @@ class InstanceSelectionResult(proto.Message):
11751224
optional=True,
11761225
)
11771226

1227+
provisioning_model_mix: ProvisioningModelMix = proto.Field(
1228+
proto.MESSAGE,
1229+
number=1,
1230+
message=ProvisioningModelMix,
1231+
)
11781232
instance_selection_list: MutableSequence[InstanceSelection] = proto.RepeatedField(
11791233
proto.MESSAGE,
11801234
number=2,
@@ -1230,6 +1284,9 @@ class DiskConfig(proto.Message):
12301284
r"""Specifies the config of disk options for a group of VM
12311285
instances.
12321286
1287+
1288+
.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields
1289+
12331290
Attributes:
12341291
boot_disk_type (str):
12351292
Optional. Type of the boot disk (default is "pd-standard").
@@ -1257,6 +1314,21 @@ class DiskConfig(proto.Message):
12571314
Valid values: "scsi" (Small Computer System Interface),
12581315
"nvme" (Non-Volatile Memory Express). See `local SSD
12591316
performance <https://cloud.google.com/compute/docs/disks/local-ssd#performance>`__.
1317+
boot_disk_provisioned_iops (int):
1318+
Optional. Indicates how many IOPS to provision for the disk.
1319+
This sets the number of I/O operations per second that the
1320+
disk can handle. Note: This field is only supported if
1321+
boot_disk_type is hyperdisk-balanced.
1322+
1323+
This field is a member of `oneof`_ ``_boot_disk_provisioned_iops``.
1324+
boot_disk_provisioned_throughput (int):
1325+
Optional. Indicates how much throughput to provision for the
1326+
disk. This sets the number of throughput mb per second that
1327+
the disk can handle. Values must be greater than or equal to
1328+
1. Note: This field is only supported if boot_disk_type is
1329+
hyperdisk-balanced.
1330+
1331+
This field is a member of `oneof`_ ``_boot_disk_provisioned_throughput``.
12601332
"""
12611333

12621334
boot_disk_type: str = proto.Field(
@@ -1275,6 +1347,16 @@ class DiskConfig(proto.Message):
12751347
proto.STRING,
12761348
number=4,
12771349
)
1350+
boot_disk_provisioned_iops: int = proto.Field(
1351+
proto.INT64,
1352+
number=5,
1353+
optional=True,
1354+
)
1355+
boot_disk_provisioned_throughput: int = proto.Field(
1356+
proto.INT64,
1357+
number=6,
1358+
optional=True,
1359+
)
12781360

12791361

12801362
class AuxiliaryNodeGroup(proto.Message):

‎packages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_cluster_controller.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6227,6 +6227,8 @@ def test_create_cluster_rest_call_success(request_type):
62276227
"boot_disk_size_gb": 1792,
62286228
"num_local_ssds": 1494,
62296229
"local_ssd_interface": "local_ssd_interface_value",
6230+
"boot_disk_provisioned_iops": 2793,
6231+
"boot_disk_provisioned_throughput": 3464,
62306232
},
62316233
"is_preemptible": True,
62326234
"preemptibility": 1,
@@ -6244,6 +6246,10 @@ def test_create_cluster_rest_call_success(request_type):
62446246
"min_cpu_platform": "min_cpu_platform_value",
62456247
"min_num_instances": 1818,
62466248
"instance_flexibility_policy": {
6249+
"provisioning_model_mix": {
6250+
"standard_capacity_base": 2296,
6251+
"standard_capacity_percent_above_base": 3764,
6252+
},
62476253
"instance_selection_list": [
62486254
{
62496255
"machine_types": [
@@ -6624,6 +6630,8 @@ def test_update_cluster_rest_call_success(request_type):
66246630
"boot_disk_size_gb": 1792,
66256631
"num_local_ssds": 1494,
66266632
"local_ssd_interface": "local_ssd_interface_value",
6633+
"boot_disk_provisioned_iops": 2793,
6634+
"boot_disk_provisioned_throughput": 3464,
66276635
},
66286636
"is_preemptible": True,
66296637
"preemptibility": 1,
@@ -6641,6 +6649,10 @@ def test_update_cluster_rest_call_success(request_type):
66416649
"min_cpu_platform": "min_cpu_platform_value",
66426650
"min_num_instances": 1818,
66436651
"instance_flexibility_policy": {
6652+
"provisioning_model_mix": {
6653+
"standard_capacity_base": 2296,
6654+
"standard_capacity_percent_above_base": 3764,
6655+
},
66446656
"instance_selection_list": [
66456657
{
66466658
"machine_types": [

‎packages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_node_group_controller.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3175,6 +3175,8 @@ def test_create_node_group_rest_call_success(request_type):
31753175
"boot_disk_size_gb": 1792,
31763176
"num_local_ssds": 1494,
31773177
"local_ssd_interface": "local_ssd_interface_value",
3178+
"boot_disk_provisioned_iops": 2793,
3179+
"boot_disk_provisioned_throughput": 3464,
31783180
},
31793181
"is_preemptible": True,
31803182
"preemptibility": 1,
@@ -3192,6 +3194,10 @@ def test_create_node_group_rest_call_success(request_type):
31923194
"min_cpu_platform": "min_cpu_platform_value",
31933195
"min_num_instances": 1818,
31943196
"instance_flexibility_policy": {
3197+
"provisioning_model_mix": {
3198+
"standard_capacity_base": 2296,
3199+
"standard_capacity_percent_above_base": 3764,
3200+
},
31953201
"instance_selection_list": [
31963202
{
31973203
"machine_types": [

‎packages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_workflow_template_service.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5865,6 +5865,8 @@ def test_create_workflow_template_rest_call_success(request_type):
58655865
"boot_disk_size_gb": 1792,
58665866
"num_local_ssds": 1494,
58675867
"local_ssd_interface": "local_ssd_interface_value",
5868+
"boot_disk_provisioned_iops": 2793,
5869+
"boot_disk_provisioned_throughput": 3464,
58685870
},
58695871
"is_preemptible": True,
58705872
"preemptibility": 1,
@@ -5882,6 +5884,10 @@ def test_create_workflow_template_rest_call_success(request_type):
58825884
"min_cpu_platform": "min_cpu_platform_value",
58835885
"min_num_instances": 1818,
58845886
"instance_flexibility_policy": {
5887+
"provisioning_model_mix": {
5888+
"standard_capacity_base": 2296,
5889+
"standard_capacity_percent_above_base": 3764,
5890+
},
58855891
"instance_selection_list": [
58865892
{
58875893
"machine_types": [
@@ -6606,6 +6612,8 @@ def test_instantiate_inline_workflow_template_rest_call_success(request_type):
66066612
"boot_disk_size_gb": 1792,
66076613
"num_local_ssds": 1494,
66086614
"local_ssd_interface": "local_ssd_interface_value",
6615+
"boot_disk_provisioned_iops": 2793,
6616+
"boot_disk_provisioned_throughput": 3464,
66096617
},
66106618
"is_preemptible": True,
66116619
"preemptibility": 1,
@@ -6623,6 +6631,10 @@ def test_instantiate_inline_workflow_template_rest_call_success(request_type):
66236631
"min_cpu_platform": "min_cpu_platform_value",
66246632
"min_num_instances": 1818,
66256633
"instance_flexibility_policy": {
6634+
"provisioning_model_mix": {
6635+
"standard_capacity_base": 2296,
6636+
"standard_capacity_percent_above_base": 3764,
6637+
},
66266638
"instance_selection_list": [
66276639
{
66286640
"machine_types": [
@@ -7097,6 +7109,8 @@ def test_update_workflow_template_rest_call_success(request_type):
70977109
"boot_disk_size_gb": 1792,
70987110
"num_local_ssds": 1494,
70997111
"local_ssd_interface": "local_ssd_interface_value",
7112+
"boot_disk_provisioned_iops": 2793,
7113+
"boot_disk_provisioned_throughput": 3464,
71007114
},
71017115
"is_preemptible": True,
71027116
"preemptibility": 1,
@@ -7114,6 +7128,10 @@ def test_update_workflow_template_rest_call_success(request_type):
71147128
"min_cpu_platform": "min_cpu_platform_value",
71157129
"min_num_instances": 1818,
71167130
"instance_flexibility_policy": {
7131+
"provisioning_model_mix": {
7132+
"standard_capacity_base": 2296,
7133+
"standard_capacity_percent_above_base": 3764,
7134+
},
71177135
"instance_selection_list": [
71187136
{
71197137
"machine_types": [

0 commit comments

Comments
 (0)