Add a new autoscaling shuffle test (ray-project#16948)

release/nightly_tests/nightly_tests.yaml

@@ -99,6 +99,16 @@
  prepare: python wait_cluster.py 4 600
  script: python shuffle/shuffle_test.py --num-partitions=200 --partition-size=500e6 --no-streaming
 
+# Test autoscaling 1TB streaming shuffle with a large number of partitions.
+- name: autoscaling_shuffle_1tb_1000_partitions
+ cluster:
+ app_config: shuffle/shuffle_app_config.yaml
+ compute_template: shuffle/shuffle_compute_autoscaling.yaml
+
+ run:
+ timeout: 3000
+ script: python shuffle/shuffle_test.py --num-partitions=1000 --partition-size=1e9 --no-streaming
+
 # Test multi nodes 1TB streaming shuffle with a large number of partitions.
 - name: shuffle_1tb_1000_partition
  cluster:

release/nightly_tests/shuffle/shuffle_compute_autoscaling.yaml

@@ -0,0 +1,19 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+aws:
+ BlockDeviceMappings:
+ - DeviceName: /dev/sda1
+ Ebs:
+ VolumeSize: 500
+
+head_node_type:
+ name: head_node
+ instance_type: i3.4xlarge
+
+worker_node_types:
+ - name: worker_node
+ instance_type: i3.4xlarge
+ min_workers: 0
+ max_workers: 19
+ use_spot: false