diff --git a/.changes/1.34.18.json b/.changes/1.34.18.json new file mode 100644 index 000000000000..ee801b19b93f --- /dev/null +++ b/.changes/1.34.18.json @@ -0,0 +1,42 @@ +[ + { + "category": "``cognito-idp``", + "description": "Added email MFA option to user pools with advanced security features.", + "type": "api-change" + }, + { + "category": "``elbv2``", + "description": "Correct incorrectly mapped error in ELBv2 waiters", + "type": "api-change" + }, + { + "category": "``emr``", + "description": "Update APIs to allow modification of ODCR options, allocation strategy, and InstanceTypeConfigs on running InstanceFleet clusters.", + "type": "api-change" + }, + { + "category": "``glue``", + "description": "AWS Glue is introducing two new optimizers for Apache Iceberg tables: snapshot retention and orphan file deletion. Customers can enable these optimizers and customize their configurations to perform daily maintenance tasks on their Iceberg tables based on their specific requirements.", + "type": "api-change" + }, + { + "category": "``mediaconvert``", + "description": "This release includes support for dynamic video overlay workflows, including picture-in-picture and squeezeback", + "type": "api-change" + }, + { + "category": "``rds``", + "description": "This release adds support for the os-upgrade pending maintenance action for Amazon Aurora DB clusters.", + "type": "api-change" + }, + { + "category": "``storagegateway``", + "description": "The S3 File Gateway now supports DSSE-KMS encryption. A new parameter EncryptionType is added to these APIs: CreateSmbFileShare, CreateNfsFileShare, UpdateSmbFileShare, UpdateNfsFileShare, DescribeSmbFileShares, DescribeNfsFileShares. Also, in favor of EncryptionType, KmsEncrypted is deprecated.", + "type": "api-change" + }, + { + "category": "``synthetics``", + "description": "This release introduces two features. The first is tag replication, which allows for the propagation of canary tags onto Synthetics related resources, such as Lambda functions. The second is a limit increase in canary name length, which has now been increased from 21 to 255 characters.", + "type": "api-change" + } +] \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 39968d019011..e367d9baf553 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,19 @@ CHANGELOG ========= +1.34.18 +======= + +* api-change:``cognito-idp``: Added email MFA option to user pools with advanced security features. +* api-change:``elbv2``: Correct incorrectly mapped error in ELBv2 waiters +* api-change:``emr``: Update APIs to allow modification of ODCR options, allocation strategy, and InstanceTypeConfigs on running InstanceFleet clusters. +* api-change:``glue``: AWS Glue is introducing two new optimizers for Apache Iceberg tables: snapshot retention and orphan file deletion. Customers can enable these optimizers and customize their configurations to perform daily maintenance tasks on their Iceberg tables based on their specific requirements. +* api-change:``mediaconvert``: This release includes support for dynamic video overlay workflows, including picture-in-picture and squeezeback +* api-change:``rds``: This release adds support for the os-upgrade pending maintenance action for Amazon Aurora DB clusters. +* api-change:``storagegateway``: The S3 File Gateway now supports DSSE-KMS encryption. A new parameter EncryptionType is added to these APIs: CreateSmbFileShare, CreateNfsFileShare, UpdateSmbFileShare, UpdateNfsFileShare, DescribeSmbFileShares, DescribeNfsFileShares. Also, in favor of EncryptionType, KmsEncrypted is deprecated. +* api-change:``synthetics``: This release introduces two features. The first is tag replication, which allows for the propagation of canary tags onto Synthetics related resources, such as Lambda functions. The second is a limit increase in canary name length, which has now been increased from 21 to 255 characters. + + 1.34.17 ======= diff --git a/awscli/__init__.py b/awscli/__init__.py index 10e3e0af3b83..9a973870f7dc 100644 --- a/awscli/__init__.py +++ b/awscli/__init__.py @@ -18,7 +18,7 @@ import os -__version__ = '1.34.17' +__version__ = '1.34.18' # # Get our data path to be added to botocore's search path diff --git a/awscli/customizations/emr/argumentschema.py b/awscli/customizations/emr/argumentschema.py index 6f86fc9f50f7..705a52a675b8 100644 --- a/awscli/customizations/emr/argumentschema.py +++ b/awscli/customizations/emr/argumentschema.py @@ -57,6 +57,41 @@ "description": "Instance group application configurations." } +ONDEMAND_CAPACITY_RESERVATION_OPTIONS_SCHEMA = { + "type": "object", + "properties" : { + "UsageStrategy": { + "type": "string", + "description": "The strategy of whether to use available capacity reservations to fulfill On-Demand capacity.", + "enum": ["use-capacity-reservations-first"] + }, + "CapacityReservationPreference": { + "type": "string", + "description": "The preference of the capacity reservation of the instance.", + "enum": [ + "open", + "none" + ] + }, + "CapacityReservationResourceGroupArn": { + "type": "string", + "description": "The ARN of the capacity reservation resource group in which to run the instance." + } + } +} + +SPOT_ALLOCATION_STRATEGY_SCHEMA = { + "type": "string", + "description": "The strategy to use to launch Spot instance fleets.", + "enum": ["capacity-optimized", "price-capacity-optimized", "lowest-price", "diversified", "capacity-optimized-prioritized"] +} + +ONDEMAND_ALLOCATION_STRATEGY_SCHEMA = { + "type": "string", + "description": "The strategy to use to launch On-Demand instance fleets.", + "enum": ["lowest-price", "prioritized"] +} + INSTANCE_GROUPS_SCHEMA = { "type": "array", "items": { @@ -411,33 +446,8 @@ "OnDemandSpecification": { "type": "object", "properties": { - "AllocationStrategy": { - "type": "string", - "description": "The strategy to use in launching On-Demand instance fleets.", - "enum": ["lowest-price", "prioritized"] - }, - "CapacityReservationOptions": { - "type": "object", - "properties" : { - "UsageStrategy": { - "type": "string", - "description": "The strategy of whether to use unused Capacity Reservations for fulfilling On-Demand capacity.", - "enum": ["use-capacity-reservations-first"] - }, - "CapacityReservationPreference": { - "type": "string", - "description": "The preference of the instance's Capacity Reservation.", - "enum": [ - "open", - "none" - ] - }, - "CapacityReservationResourceGroupArn": { - "type": "string", - "description": "The ARN of the Capacity Reservation resource group in which to run the instance." - } - } - } + "AllocationStrategy": ONDEMAND_ALLOCATION_STRATEGY_SCHEMA, + "CapacityReservationOptions": ONDEMAND_CAPACITY_RESERVATION_OPTIONS_SCHEMA } }, "SpotSpecification": { @@ -459,11 +469,7 @@ "type": "integer", "description": "Block duration in minutes." }, - "AllocationStrategy": { - "type": "string", - "description": "The strategy to use in launching Spot instance fleets.", - "enum": ["capacity-optimized", "price-capacity-optimized", "lowest-price", "diversified", "capacity-optimized-prioritized"] - } + "AllocationStrategy": SPOT_ALLOCATION_STRATEGY_SCHEMA } } } @@ -477,7 +483,8 @@ "TimeoutDurationMinutes": { "type" : "integer", "description": "The time, in minutes, after which the resize will be stopped if requested resources are unavailable." - } + }, + "AllocationStrategy": SPOT_ALLOCATION_STRATEGY_SCHEMA } }, "OnDemandResizeSpecification": { @@ -486,7 +493,9 @@ "TimeoutDurationMinutes": { "type" : "integer", "description": "The time, in minutes, after which the resize will be stopped if requested resources are unavailable." - } + }, + "AllocationStrategy": ONDEMAND_ALLOCATION_STRATEGY_SCHEMA, + "CapacityReservationOptions": ONDEMAND_CAPACITY_RESERVATION_OPTIONS_SCHEMA } } } diff --git a/doc/source/conf.py b/doc/source/conf.py index bc3a08c8073d..aaac529fe9b4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.34.' # The full version, including alpha/beta/rc tags. -release = '1.34.17' +release = '1.34.18' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.cfg b/setup.cfg index ebd5fffc6bdd..d848c14c02ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ universal = 0 [metadata] requires_dist = - botocore==1.35.17 + botocore==1.35.18 docutils>=0.10,<0.17 s3transfer>=0.10.0,<0.11.0 PyYAML>=3.10,<6.1 diff --git a/setup.py b/setup.py index e98d87ecfcf0..a7d2561b8f38 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def find_version(*file_paths): install_requires = [ - 'botocore==1.35.17', + 'botocore==1.35.18', 'docutils>=0.10,<0.17', 's3transfer>=0.10.0,<0.11.0', 'PyYAML>=3.10,<6.1', diff --git a/tests/unit/customizations/emr/test_add_instance_fleet.py b/tests/unit/customizations/emr/test_add_instance_fleet.py new file mode 100644 index 000000000000..d4a658ebc21c --- /dev/null +++ b/tests/unit/customizations/emr/test_add_instance_fleet.py @@ -0,0 +1,33 @@ +# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from tests.unit.customizations.emr import EMRBaseAWSCommandParamsTest as \ + BaseAWSCommandParamsTest +from tests.unit.customizations.emr import test_constants_instance_fleets as \ + CONSTANTS_FLEET + +class TestAddInstanceFleet(BaseAWSCommandParamsTest): + prefix = f"emr add-instance-fleet --cluster-id {CONSTANTS_FLEET.DEFAULT_CLUSTER_NAME} --instance-fleet " + + def test_add_instance_fleet_with_allocation_strategy_spot_and_od(self): + result = \ + { + "ClusterId": CONSTANTS_FLEET.DEFAULT_CLUSTER_NAME, + "InstanceFleet": CONSTANTS_FLEET.RES_TASK_INSTANCE_FLEET_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD + } + self.assert_params_for_cmd( + self.prefix + CONSTANTS_FLEET.TASK_INSTANCE_FLEET_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD, + result) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/customizations/emr/test_constants_instance_fleets.py b/tests/unit/customizations/emr/test_constants_instance_fleets.py index ed92e8fa5e79..f90c616bb1ee 100644 --- a/tests/unit/customizations/emr/test_constants_instance_fleets.py +++ b/tests/unit/customizations/emr/test_constants_instance_fleets.py @@ -11,6 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +DEFAULT_INSTANCE_FLEET_NAME = "if-XYZ123" +DEFAULT_CLUSTER_NAME = "j-ABC123456" + INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY = ( 'InstanceFleetType=MASTER,TargetOnDemandCapacity=1,InstanceTypeConfigs=[{InstanceType=d2.xlarge}],' 'LaunchSpecifications={OnDemandSpecification={AllocationStrategy=lowest-price,' @@ -70,17 +73,66 @@ 'TimeoutAction=TERMINATE_CLUSTER,AllocationStrategy=diversified}}') INSTANCE_FLEETS_WITH_PRIORITIZED_ALLOCATION_STRATEGY_SPOT_AND_OD = ( - 'InstanceFleetType=MASTER,TargetSpotCapacity=1,InstanceTypeConfigs=[{InstanceType=d2.xlarge,BidPrice=0.1,Priority=0.0}],' - 'LaunchSpecifications={SpotSpecification={TimeoutDurationMinutes=30,TimeoutAction=TERMINATE_CLUSTER,' - 'AllocationStrategy=capacity-optimized-prioritized},OnDemandSpecification={AllocationStrategy=prioritized}} ' - 'InstanceFleetType=CORE,TargetSpotCapacity=100,InstanceTypeConfigs=[{InstanceType=d2.xlarge,BidPrice=0.5,' - 'WeightedCapacity=1,Priority=0.0},{InstanceType=m3.2xlarge,BidPrice=0.2,WeightedCapacity=2,Priority=1.0},{InstanceType=m3.4xlarge,BidPrice=0.4,' - 'WeightedCapacity=4,Priority=99.0}],LaunchSpecifications={SpotSpecification={TimeoutDurationMinutes=32,' - 'TimeoutAction=TERMINATE_CLUSTER,AllocationStrategy=capacity-optimized-prioritized},OnDemandSpecification={AllocationStrategy=prioritized}} ' - 'InstanceFleetType=TASK,TargetSpotCapacity=100,InstanceTypeConfigs=[{InstanceType=d2.xlarge,BidPrice=0.5,' - 'WeightedCapacity=1,Priority=10.0},{InstanceType=m3.2xlarge,BidPrice=0.2,WeightedCapacity=2,Priority=0.0},{InstanceType=m3.4xlarge,BidPrice=0.4,' - 'WeightedCapacity=4,Priority=100.0}],LaunchSpecifications={SpotSpecification={TimeoutDurationMinutes=77,' - 'TimeoutAction=TERMINATE_CLUSTER,AllocationStrategy=capacity-optimized-prioritized},OnDemandSpecification={AllocationStrategy=prioritized}}') + 'InstanceFleetType=MASTER,TargetSpotCapacity=1,InstanceTypeConfigs=[{InstanceType=d2.xlarge,' + 'BidPrice=0.1,Priority=0.0}],LaunchSpecifications={SpotSpecification={TimeoutDurationMinutes=30,' + 'TimeoutAction=TERMINATE_CLUSTER,AllocationStrategy=capacity-optimized-prioritized},' + 'OnDemandSpecification={AllocationStrategy=prioritized}} ' + 'InstanceFleetType=CORE,TargetSpotCapacity=100,InstanceTypeConfigs=[{InstanceType=d2.xlarge,' + 'BidPrice=0.5,WeightedCapacity=1,Priority=0.0},{InstanceType=m3.2xlarge,BidPrice=0.2,' + 'WeightedCapacity=2,Priority=1.0},{InstanceType=m3.4xlarge,BidPrice=0.4,WeightedCapacity=4,' + 'Priority=99.0}],LaunchSpecifications={SpotSpecification={TimeoutDurationMinutes=32,' + 'TimeoutAction=TERMINATE_CLUSTER,AllocationStrategy=capacity-optimized-prioritized},' + 'OnDemandSpecification={AllocationStrategy=prioritized}} ' + 'InstanceFleetType=TASK,TargetSpotCapacity=100,InstanceTypeConfigs=[{InstanceType=d2.xlarge,' + 'BidPrice=0.5,WeightedCapacity=1,Priority=10.0},{InstanceType=m3.2xlarge,BidPrice=0.2,' + 'WeightedCapacity=2,Priority=0.0},{InstanceType=m3.4xlarge,BidPrice=0.4,WeightedCapacity=4,' + 'Priority=100.0}],LaunchSpecifications={SpotSpecification={TimeoutDurationMinutes=77,' + 'TimeoutAction=TERMINATE_CLUSTER,AllocationStrategy=' + 'capacity-optimized-prioritized},OnDemandSpecification={AllocationStrategy=prioritized}}') + +TASK_INSTANCE_FLEET_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD = ( + 'InstanceFleetType=TASK,TargetSpotCapacity=100,InstanceTypeConfigs=[{InstanceType=d2.xlarge,' + 'BidPrice=0.5,WeightedCapacity=1},{InstanceType=m3.2xlarge,BidPrice=0.2,WeightedCapacity=2},' + '{InstanceType=m3.4xlarge,BidPrice=0.4,WeightedCapacity=4}],LaunchSpecifications={' + 'SpotSpecification={TimeoutDurationMinutes=77,TimeoutAction=TERMINATE_CLUSTER,' + 'AllocationStrategy=capacity-optimized-prioritized},OnDemandSpecification={' + 'AllocationStrategy=lowest-price}},ResizeSpecifications={SpotResizeSpecification={' + 'AllocationStrategy=capacity-optimized},OnDemandResizeSpecification={' + 'AllocationStrategy=lowest-price,CapacityReservationOptions={' + 'UsageStrategy=use-capacity-reservations-first,CapacityReservationPreference=open}}}') + +INSTANCE_FLEETS_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD = ( + 'InstanceFleetType=MASTER,TargetSpotCapacity=1,InstanceTypeConfigs=[{InstanceType=d2.xlarge,' + 'BidPrice=0.1}],LaunchSpecifications={SpotSpecification={TimeoutDurationMinutes=30,' + 'TimeoutAction=TERMINATE_CLUSTER,AllocationStrategy=capacity-optimized-prioritized},' + 'OnDemandSpecification={AllocationStrategy=lowest-price}} ' + 'InstanceFleetType=CORE,TargetSpotCapacity=100,InstanceTypeConfigs=[{InstanceType=d2.xlarge,' + 'BidPrice=0.5,WeightedCapacity=1},{InstanceType=m3.2xlarge,BidPrice=0.2,WeightedCapacity=2},' + '{InstanceType=m3.4xlarge,BidPrice=0.4,WeightedCapacity=4}],LaunchSpecifications={' + 'SpotSpecification={TimeoutDurationMinutes=32,TimeoutAction=TERMINATE_CLUSTER,' + 'AllocationStrategy=capacity-optimized-prioritized},OnDemandSpecification={' + 'AllocationStrategy=lowest-price}},ResizeSpecifications={SpotResizeSpecification=' + '{AllocationStrategy=capacity-optimized},OnDemandResizeSpecification={' + 'AllocationStrategy=lowest-price,CapacityReservationOptions={' + 'UsageStrategy=use-capacity-reservations-first,CapacityReservationPreference=open}}} ' + f'{TASK_INSTANCE_FLEET_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD}') + +MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS = ( + f'InstanceFleetId={DEFAULT_INSTANCE_FLEET_NAME},' + f'InstanceTypeConfigs=[{{InstanceType=d2.xlarge}}]') + +MODIFY_INSTANCE_FLEET_WITH_SPOT_AND_OD_RESIZE_SPECIFICATIONS = ( + f'InstanceFleetId={DEFAULT_INSTANCE_FLEET_NAME},ResizeSpecifications={{SpotResizeSpecification=' + f'{{AllocationStrategy=capacity-optimized}},OnDemandResizeSpecification={{' + f'AllocationStrategy=lowest-price,CapacityReservationOptions={{' + f'UsageStrategy=use-capacity-reservations-first,CapacityReservationPreference=open}}}}}}') + +MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS_AND_SPOT_AND_OD_RESIZE_SPECIFICATIONS = ( + f'InstanceFleetId={DEFAULT_INSTANCE_FLEET_NAME},ResizeSpecifications={{SpotResizeSpecification=' + f'{{AllocationStrategy=capacity-optimized}},OnDemandResizeSpecification={{' + f'AllocationStrategy=lowest-price,CapacityReservationOptions={{' + f'UsageStrategy=use-capacity-reservations-first,CapacityReservationPreference=open}}}}}}' + f',InstanceTypeConfigs=[{{InstanceType=d2.xlarge}}]') RES_INSTANCE_FLEETS_WITH_ON_DEMAND_MASTER_ONLY = \ [{"InstanceTypeConfigs": [{"InstanceType": "d2.xlarge"}], @@ -256,36 +308,305 @@ "Name": "TASK" }] - RES_INSTANCE_FLEETS_WITH_PRIORITIZED_ALLOCATION_STRATEGY_SPOT_AND_OD = \ - [{"InstanceTypeConfigs": [{"InstanceType": "d2.xlarge","BidPrice": "0.1","Priority": 0.0}], + [ + { + "InstanceTypeConfigs": [ + { + "InstanceType": "d2.xlarge", + "BidPrice": "0.1", + "Priority": 0 + } + ], "LaunchSpecifications": { - "SpotSpecification": {"TimeoutDurationMinutes": 30, "TimeoutAction": "TERMINATE_CLUSTER", "AllocationStrategy": "capacity-optimized-prioritized"}, - "OnDemandSpecification": {"AllocationStrategy": "prioritized"} + "SpotSpecification": { + "TimeoutDurationMinutes": 30, + "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized" + }, + "OnDemandSpecification": { + "AllocationStrategy": "prioritized" + } }, "TargetSpotCapacity": 1, "InstanceFleetType": "MASTER", "Name": "MASTER" }, - {"InstanceTypeConfigs": [{"InstanceType": "d2.xlarge","BidPrice": "0.5","WeightedCapacity": 1,"Priority": 0.0}, - {"InstanceType": "m3.2xlarge","BidPrice": "0.2","WeightedCapacity": 2,"Priority": 1.0},{"InstanceType": "m3.4xlarge","BidPrice": "0.4", - "WeightedCapacity": 4,"Priority": 99.0}], - "LaunchSpecifications" : { - "SpotSpecification": {"TimeoutDurationMinutes": 32, "TimeoutAction": "TERMINATE_CLUSTER", "AllocationStrategy": "capacity-optimized-prioritized"}, - "OnDemandSpecification": {"AllocationStrategy": "prioritized"} + { + "InstanceTypeConfigs": [ + { + "InstanceType": "d2.xlarge", + "BidPrice": "0.5", + "WeightedCapacity": 1, + "Priority": 0 + }, + { + "InstanceType": "m3.2xlarge", + "BidPrice": "0.2", + "WeightedCapacity": 2, + "Priority": 1 + }, + { + "InstanceType": "m3.4xlarge", + "BidPrice": "0.4", + "WeightedCapacity": 4, + "Priority": 99 + } + ], + "LaunchSpecifications": { + "SpotSpecification": { + "TimeoutDurationMinutes": 32, + "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized" + }, + "OnDemandSpecification": { + "AllocationStrategy": "prioritized" + } }, "TargetSpotCapacity": 100, "InstanceFleetType": "CORE", "Name": "CORE" }, - {"InstanceTypeConfigs": [{"InstanceType": "d2.xlarge","BidPrice": "0.5","WeightedCapacity": 1,"Priority": 10.0}, - {"InstanceType": "m3.2xlarge","BidPrice": "0.2","WeightedCapacity": 2,"Priority": 0.0},{"InstanceType": "m3.4xlarge","BidPrice": "0.4", - "WeightedCapacity": 4,"Priority": 100.0}], - "LaunchSpecifications" : { - "SpotSpecification": {"TimeoutDurationMinutes": 77, "TimeoutAction": "TERMINATE_CLUSTER", "AllocationStrategy": "capacity-optimized-prioritized"}, - "OnDemandSpecification": {"AllocationStrategy": "prioritized"} + { + "InstanceTypeConfigs": [ + { + "InstanceType": "d2.xlarge", + "BidPrice": "0.5", + "WeightedCapacity": 1, + "Priority": 10 + }, + { + "InstanceType": "m3.2xlarge", + "BidPrice": "0.2", + "WeightedCapacity": 2, + "Priority": 0 + }, + { + "InstanceType": "m3.4xlarge", + "BidPrice": "0.4", + "WeightedCapacity": 4, + "Priority": 100 + } + ], + "LaunchSpecifications": { + "SpotSpecification": { + "TimeoutDurationMinutes": 77, + "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized" + }, + "OnDemandSpecification": { + "AllocationStrategy": "prioritized" + } }, "TargetSpotCapacity": 100, "InstanceFleetType": "TASK", "Name": "TASK" - }] \ No newline at end of file + }] + +RES_INSTANCE_FLEETS_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD = \ + [ + { + "InstanceTypeConfigs": [ + { + "InstanceType": "d2.xlarge", + "BidPrice": "0.1" + } + ], + "LaunchSpecifications": { + "SpotSpecification": { + "TimeoutDurationMinutes": 30, + "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized" + }, + "OnDemandSpecification": { + "AllocationStrategy": "lowest-price" + } + }, + "TargetSpotCapacity": 1, + "InstanceFleetType": "MASTER", + "Name": "MASTER" + }, + { + "InstanceTypeConfigs": [ + { + "InstanceType": "d2.xlarge", + "BidPrice": "0.5", + "WeightedCapacity": 1 + }, + { + "InstanceType": "m3.2xlarge", + "BidPrice": "0.2", + "WeightedCapacity": 2 + }, + { + "InstanceType": "m3.4xlarge", + "BidPrice": "0.4", + "WeightedCapacity": 4 + } + ], + "LaunchSpecifications": { + "SpotSpecification": { + "TimeoutDurationMinutes": 32, + "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized" + }, + "OnDemandSpecification": { + "AllocationStrategy": "lowest-price" + } + }, + "ResizeSpecifications": { + "OnDemandResizeSpecification": { + "AllocationStrategy": "lowest-price", + "CapacityReservationOptions": { + "CapacityReservationPreference": "open", + "UsageStrategy": "use-capacity-reservations-first" + } + }, + "SpotResizeSpecification": { + "AllocationStrategy": "capacity-optimized" + } + }, + "TargetSpotCapacity": 100, + "InstanceFleetType": "CORE", + "Name": "CORE" + }, + { + "InstanceTypeConfigs": [ + { + "InstanceType": "d2.xlarge", + "BidPrice": "0.5", + "WeightedCapacity": 1 + }, + { + "InstanceType": "m3.2xlarge", + "BidPrice": "0.2", + "WeightedCapacity": 2 + }, + { + "InstanceType": "m3.4xlarge", + "BidPrice": "0.4", + "WeightedCapacity": 4 + } + ], + "LaunchSpecifications": { + "SpotSpecification": { + "TimeoutDurationMinutes": 77, + "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized" + }, + "OnDemandSpecification": { + "AllocationStrategy": "lowest-price" + } + }, + "ResizeSpecifications": { + "OnDemandResizeSpecification": { + "AllocationStrategy": "lowest-price", + "CapacityReservationOptions": { + "CapacityReservationPreference": "open", + "UsageStrategy": "use-capacity-reservations-first" + } + }, + "SpotResizeSpecification": { + "AllocationStrategy": "capacity-optimized" + } + }, + "TargetSpotCapacity": 100, + "InstanceFleetType": "TASK", + "Name": "TASK" + } + ] + +RES_TASK_INSTANCE_FLEET_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD = \ + { + "InstanceTypeConfigs": [ + { + "InstanceType": "d2.xlarge", + "BidPrice": "0.5", + "WeightedCapacity": 1 + }, + { + "InstanceType": "m3.2xlarge", + "BidPrice": "0.2", + "WeightedCapacity": 2 + }, + { + "InstanceType": "m3.4xlarge", + "BidPrice": "0.4", + "WeightedCapacity": 4 + } + ], + "LaunchSpecifications": { + "SpotSpecification": { + "TimeoutDurationMinutes": 77, + "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized" + }, + "OnDemandSpecification": { + "AllocationStrategy": "lowest-price" + } + }, + "ResizeSpecifications": { + "OnDemandResizeSpecification": { + "AllocationStrategy": "lowest-price", + "CapacityReservationOptions": { + "CapacityReservationPreference": "open", + "UsageStrategy": "use-capacity-reservations-first" + } + }, + "SpotResizeSpecification": { + "AllocationStrategy": "capacity-optimized" + } + }, + "TargetSpotCapacity": 100, + "InstanceFleetType": "TASK" + } + +RES_MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS = \ + { + "ClusterId": DEFAULT_CLUSTER_NAME, + "InstanceFleet": { + "InstanceFleetId": DEFAULT_INSTANCE_FLEET_NAME, + "InstanceTypeConfigs": [ + {"InstanceType": "d2.xlarge"} + ] + } + } + +RES_MODIFY_INSTANCE_FLEET_WITH_SPOT_AND_OD_RESIZE_SPECIFICATIONS = \ + { + "ClusterId": DEFAULT_CLUSTER_NAME, + "InstanceFleet": { + "InstanceFleetId": DEFAULT_INSTANCE_FLEET_NAME, + "ResizeSpecifications": { + "OnDemandResizeSpecification": { + "AllocationStrategy": "lowest-price", + "CapacityReservationOptions": { + "CapacityReservationPreference": "open", + "UsageStrategy": "use-capacity-reservations-first" + } + }, + "SpotResizeSpecification": {"AllocationStrategy": "capacity-optimized"} + } + } + } + +RES_MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS_AND_SPOT_AND_OD_RESIZE_SPECIFICATIONS = \ + { + "ClusterId": DEFAULT_CLUSTER_NAME, + "InstanceFleet": { + "InstanceFleetId": DEFAULT_INSTANCE_FLEET_NAME, + "ResizeSpecifications": { + "OnDemandResizeSpecification": { + "AllocationStrategy": "lowest-price", + "CapacityReservationOptions": { + "CapacityReservationPreference": "open", + "UsageStrategy": "use-capacity-reservations-first" + } + }, + "SpotResizeSpecification": {"AllocationStrategy": "capacity-optimized"} + }, + "InstanceTypeConfigs": [ + {"InstanceType": "d2.xlarge"} + ] + } + } \ No newline at end of file diff --git a/tests/unit/customizations/emr/test_create_cluster_release_label.py b/tests/unit/customizations/emr/test_create_cluster_release_label.py index 14dc4969a84f..5fef58312559 100644 --- a/tests/unit/customizations/emr/test_create_cluster_release_label.py +++ b/tests/unit/customizations/emr/test_create_cluster_release_label.py @@ -1552,6 +1552,26 @@ def test_create_cluster_with_placement_groups(self): } self.assert_params_for_cmd(cmd, result) + def test_instance_fleets_with_resize_allocation_strategy_spot_od(self): + cmd = (self.prefix + '--release-label emr-4.2.0 --instance-fleets ' + + CONSTANTS_FLEET.INSTANCE_FLEETS_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD + + ' --ec2-attributes AvailabilityZones=[us-east-1a,us-east-1b]') + instance_fleets = CONSTANTS_FLEET.RES_INSTANCE_FLEETS_WITH_RESIZE_ALLOCATION_STRATEGY_SPOT_AND_OD + result = \ + { + 'Name': DEFAULT_CLUSTER_NAME, + 'Instances': {'KeepJobFlowAliveWhenNoSteps': True, + 'TerminationProtected': False, + 'InstanceFleets': + instance_fleets, + 'Placement': {'AvailabilityZones': ['us-east-1a','us-east-1b']} + }, + 'ReleaseLabel': 'emr-4.2.0', + 'VisibleToAllUsers': True, + 'Tags': [] + } + self.assert_params_for_cmd(cmd, result) + def test_create_cluster_with_os_release_label(self): test_os_release_label = '2.0.20220406.1' cmd = (self.prefix + '--release-label emr-6.6.0' diff --git a/tests/unit/customizations/emr/test_describe_cluster.py b/tests/unit/customizations/emr/test_describe_cluster.py index 7194e0cd5a45..eb13d02f37dd 100644 --- a/tests/unit/customizations/emr/test_describe_cluster.py +++ b/tests/unit/customizations/emr/test_describe_cluster.py @@ -176,6 +176,20 @@ "BidPriceAsPercentageOfOnDemandPrice": 0.0 } ], + "LaunchSpecifications" : { + "SpotSpecification": {"TimeoutDurationMinutes": 77, "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized"}, + "OnDemandSpecification": {"AllocationStrategy": "lowest-price"} + }, + "ResizeSpecifications": { + "OnDemandResizeSpecification": {"AllocationStrategy": "lowest-price", + "CapacityReservationOptions": { + "CapacityReservationPreference": "open", + "UsageStrategy": "use-capacity-reservations-first" + } + }, + "SpotResizeSpecification": {"AllocationStrategy": "capacity-optimized"} + }, "Name": "Master instance group", "InstanceFleetType": "MASTER", "InstanceType": "m1.large", @@ -327,6 +341,20 @@ "BidPriceAsPercentageOfOnDemandPrice": 0.0 } ], + "LaunchSpecifications" : { + "SpotSpecification": {"TimeoutDurationMinutes": 77, "TimeoutAction": "TERMINATE_CLUSTER", + "AllocationStrategy": "capacity-optimized-prioritized"}, + "OnDemandSpecification": {"AllocationStrategy": "lowest-price"} + }, + "ResizeSpecifications": { + "OnDemandResizeSpecification": {"AllocationStrategy": "lowest-price", + "CapacityReservationOptions": { + "CapacityReservationPreference": "open", + "UsageStrategy": "use-capacity-reservations-first" + } + }, + "SpotResizeSpecification": {"AllocationStrategy": "capacity-optimized"} + }, "Name": "Master instance group", "InstanceFleetType": "MASTER", "InstanceType": "m1.large", diff --git a/tests/unit/customizations/emr/test_modify_instance_fleet.py b/tests/unit/customizations/emr/test_modify_instance_fleet.py new file mode 100644 index 000000000000..418406acd5a0 --- /dev/null +++ b/tests/unit/customizations/emr/test_modify_instance_fleet.py @@ -0,0 +1,40 @@ +# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from tests.unit.customizations.emr import EMRBaseAWSCommandParamsTest as \ + BaseAWSCommandParamsTest +from tests.unit.customizations.emr import test_constants_instance_fleets as \ + CONSTANTS_FLEET + + +class TestModifyInstanceFleet(BaseAWSCommandParamsTest): + prefix = f"emr modify-instance-fleet --cluster-id {CONSTANTS_FLEET.DEFAULT_CLUSTER_NAME} --instance-fleet " + + def test_modify_instance_fleet_with_instance_type_configs(self): + self.assert_params_for_cmd( + self.prefix + CONSTANTS_FLEET.MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS, + CONSTANTS_FLEET.RES_MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS) + + def test_modify_instance_fleet_with_allocation_strategy_spot_and_od(self): + self.assert_params_for_cmd( + self.prefix + CONSTANTS_FLEET.MODIFY_INSTANCE_FLEET_WITH_SPOT_AND_OD_RESIZE_SPECIFICATIONS, + CONSTANTS_FLEET.RES_MODIFY_INSTANCE_FLEET_WITH_SPOT_AND_OD_RESIZE_SPECIFICATIONS) + + def test_modify_instance_fleet_with_allocation_strategy_spot_and_od_and_instance_type_configs(self): + self.assert_params_for_cmd( + self.prefix + CONSTANTS_FLEET.MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS_AND_SPOT_AND_OD_RESIZE_SPECIFICATIONS, + CONSTANTS_FLEET.RES_MODIFY_INSTANCE_FLEET_WITH_INSTANCE_TYPE_CONFIGS_AND_SPOT_AND_OD_RESIZE_SPECIFICATIONS) + + +if __name__ == "__main__": + unittest.main()