diff --git a/release/nightly_tests/nightly_tests.yaml b/release/nightly_tests/nightly_tests.yaml index acb1496aa3cd0..c8b4b6730da68 100644 --- a/release/nightly_tests/nightly_tests.yaml +++ b/release/nightly_tests/nightly_tests.yaml @@ -99,6 +99,16 @@ prepare: python wait_cluster.py 4 600 script: python shuffle/shuffle_test.py --num-partitions=200 --partition-size=500e6 --no-streaming +# Test autoscaling 1TB streaming shuffle with a large number of partitions. +- name: autoscaling_shuffle_1tb_1000_partitions + cluster: + app_config: shuffle/shuffle_app_config.yaml + compute_template: shuffle/shuffle_compute_autoscaling.yaml + + run: + timeout: 3000 + script: python shuffle/shuffle_test.py --num-partitions=1000 --partition-size=1e9 --no-streaming + # Test multi nodes 1TB streaming shuffle with a large number of partitions. - name: shuffle_1tb_1000_partition cluster: diff --git a/release/nightly_tests/shuffle/shuffle_compute_autoscaling.yaml b/release/nightly_tests/shuffle/shuffle_compute_autoscaling.yaml new file mode 100644 index 0000000000000..a1ad5a7b2dad8 --- /dev/null +++ b/release/nightly_tests/shuffle/shuffle_compute_autoscaling.yaml @@ -0,0 +1,19 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +aws: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: 500 + +head_node_type: + name: head_node + instance_type: i3.4xlarge + +worker_node_types: + - name: worker_node + instance_type: i3.4xlarge + min_workers: 0 + max_workers: 19 + use_spot: false