Merge remote-tracking branch 'origin/main' into multi-ds-eval

elk/extraction/extraction.py

@@ -326,7 +326,7 @@ def get_splits() -> SplitDict:
  split_name=split_name,
  split_info=split_info,
  gen_kwargs=dict(
- cfg=[_cfg] * len(devices),
+ cfg=[cfg] * len(devices),
  device=devices,
  rank=list(range(len(devices))),
  split_type=[split_name] * len(devices),

elk/extraction/generator.py

@@ -34,22 +34,6 @@ def create_config_id(
  }
  return super().create_config_id(config_kwargs, custom_features)
 
- def create_config_id(
- self, config_kwargs: dict, custom_features: Features | None
- ) -> str:
- config_kwargs = deepcopy(config_kwargs)
-
- # By default the values in gen_kwargs are lists of length world_size. We want
- # to erase the world_size dimension so that the config id is the same no matter
- # how many processes are used. We also remove the explicit device, rank, and
- # world_size keys.
- config_kwargs["gen_kwargs"] = {
- k: v[0]
- for k, v in config_kwargs.get("gen_kwargs", {}).items()
- if k not in ("device", "rank", "world_size")
- }
- return super().create_config_id(config_kwargs, custom_features)
-
 
 @dataclass
 class _SplitGenerator: