Skip to content

Commit

Permalink
[Data] Avoid warning of compute argument in add/select/drop_columns() (
Browse files Browse the repository at this point in the history
…ray-project#43392)

This PR is to fix the bug when user uses `add/select/drop_columns` without `compute`/`concurrency` API. Before this PR, it prints out a deprecation warning by mistake. This PR is to avoid printing out the warning.

Signed-off-by: Cheng Su <[email protected]>
  • Loading branch information
c21 committed Feb 23, 2024
1 parent c328809 commit 852564c
Showing 1 changed file with 3 additions and 15 deletions.
18 changes: 3 additions & 15 deletions python/ray/data/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -646,11 +646,6 @@ def add_column(
ray_remote_args: Additional resource requirements to request from
ray (e.g., num_gpus=1 to request GPUs for the map tasks).
"""
compute = get_compute_strategy(
fn,
compute=compute,
concurrency=concurrency,
)

def process_batch(batch: "pandas.DataFrame") -> "pandas.DataFrame":
batch.loc[:, col] = fn(batch)
Expand All @@ -663,6 +658,7 @@ def process_batch(batch: "pandas.DataFrame") -> "pandas.DataFrame":
process_batch,
batch_format="pandas", # TODO(ekl) we should make this configurable.
compute=compute,
concurrency=concurrency,
zero_copy_batch=False,
**ray_remote_args,
)
Expand Down Expand Up @@ -713,16 +709,12 @@ def drop_columns(
def fn(batch):
return batch.drop(columns=cols)

compute = get_compute_strategy(
fn,
compute=compute,
concurrency=concurrency,
)
return self.map_batches(
fn,
batch_format="pandas",
zero_copy_batch=True,
compute=compute,
concurrency=concurrency,
**ray_remote_args,
)

Expand Down Expand Up @@ -772,16 +764,12 @@ def select_columns(
def fn(batch):
return BlockAccessor.for_block(batch).select(columns=cols)

compute = get_compute_strategy(
fn,
compute=compute,
concurrency=concurrency,
)
return self.map_batches(
fn,
batch_format="pandas",
zero_copy_batch=True,
compute=compute,
concurrency=concurrency,
**ray_remote_args,
)

Expand Down

0 comments on commit 852564c

Please sign in to comment.