add progress bar to http_get

Signed-off-by: Weixiong Lin <[email protected]>

scispacy/file_cache.py

@@ -12,6 +12,7 @@
 from hashlib import sha256
 
 import requests
+from tqdm import tqdm
 
 CACHE_ROOT = Path(os.getenv("SCISPACY_CACHE", str(Path.home() / ".scispacy")))
 DATASET_CACHE = str(CACHE_ROOT / "datasets")
@@ -96,9 +97,14 @@ def filename_to_url(filename: str, cache_dir: Optional[str] = None) -> Tuple[str
 
 def http_get(url: str, temp_file: IO) -> None:
  req = requests.get(url, stream=True)
+ total = int(req.headers.get("content-length", 0))
+ pbar = tqdm(total=total, unit="iB", unit_scale=True, unit_divisor=1024)
  for chunk in req.iter_content(chunk_size=1024):
  if chunk: # filter out keep-alive new chunks
  temp_file.write(chunk)
+ size = temp_file.write(chunk)
+ pbar.update(size)
+ pbar.close()
 
 
 def get_from_cache(url: str, cache_dir: Optional[str] = None) -> str: