Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Simple change to fix openai#1394 .
  • Loading branch information
Lorenzo Pacchiardi committed Jan 3, 2024
1 parent 0647721 commit 10b02c6
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions evals/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,16 @@ def zstd_open(filename: str, mode: str = "rb", openhook: Any = open) -> pyzstd.Z

return pyzstd.ZstdFile(openhook(filename, mode), mode=mode)


def open_by_file_pattern(filename: str, mode: str = "r", **kwargs: Any) -> Any:
def open_by_file_pattern(filename: Union[str, Path], mode: str = "r", **kwargs: Any) -> Any:
"""Can read/write to files on gcs/local with or without gzipping. If file
is stored on gcs, streams with blobfile. Otherwise use vanilla python open. If
filename endswith gz, then zip/unzip contents on the fly (note that gcs paths and
gzip are compatible)"""
open_fn = partial(bf.BlobFile, **kwargs)

if isinstance(filename, Path):
filename = filename.as_posix()

try:
if filename.endswith(".gz"):
return gzip_open(filename, openhook=open_fn, mode=mode)
Expand Down

0 comments on commit 10b02c6

Please sign in to comment.