move from urllib to request, move donwload under each class

.github/workflows/build.yml

@@ -46,7 +46,6 @@ jobs:
  DUST_DIR: ${{github.workspace}}/data/
  run: |
  python setup.py develop
- pip install requests
  - name: Test package
  env:
  DUST_DIR: ${{github.workspace}}/data/

mwdust/Combined15.py

@@ -7,7 +7,7 @@
 import os, os.path
 import numpy
 import h5py
-from mwdust.DustMap3D import dust_dir
+from mwdust.DustMap3D import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
 _combineddir= os.path.join(dust_dir, 'combined15')
@@ -48,3 +48,12 @@ def __init__(self,filter=None,sf10=True,load_samples=False,
  dtype='object') #array to cache interpolated extinctions
  self._interpk= interpk
  return None
+ @classmethod
+ def download(cls, test=False):
+ # Download the combined map of Bovy et al. (2015): Marshall+Green+Drimmel for full sky coverage
+ combined15_path = os.path.join(dust_dir, "combined15", "dust-map-3d.h5")
+ if not os.path.exists(combined15_path):
+ if not os.path.exists(os.path.join(dust_dir, "combined15")):
+ os.mkdir(os.path.join(dust_dir, "combined15"))
+ _COMBINED15_URL = "https://zenodo.org/record/31262/files/dust-map-3d.h5"
+ downloader(_COMBINED15_URL, combined15_path, "COMBINED15", test=test)

mwdust/Combined19.py

@@ -7,7 +7,7 @@
 import os, os.path
 import numpy
 import h5py
-from mwdust.DustMap3D import dust_dir
+from mwdust.DustMap3D import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
 _combineddir= os.path.join(dust_dir,'combined19')
@@ -49,3 +49,13 @@ def __init__(self,filter=None,sf10=True,load_samples=False,
  dtype='object') #array to cache interpolated extinctions
  self._interpk= interpk
  return None
+
+ @classmethod
+ def download(cls, test=False):
+ # Download the combined map: Marshall+Green19+Drimmel for full sky coverage
+ combined19_path = os.path.join(dust_dir, "combined19", "combine19.h5")
+ if not os.path.exists(combined19_path):
+ if not os.path.exists(os.path.join(dust_dir, "combined19")):
+ os.mkdir(os.path.join(dust_dir, "combined19"))
+ _COMBINED19_URL = "https://zenodo.org/record/3566060/files/combine19.h5"
+ downloader(_COMBINED19_URL, combined19_path, "COMBINED19", test=test)

mwdust/Drimmel03.py

@@ -3,8 +3,11 @@
 # Drimmel03: extinction model from Drimmel et al. 2003 2003A&A...409..205D
 #
 ###############################################################################
+import os
 import copy
 import numpy
+import tarfile
+import inspect
 from scipy.ndimage import map_coordinates
 from scipy import optimize
 try:
@@ -13,7 +16,8 @@
 from mwdust.util.extCurves import aebv
 from mwdust.util import read_Drimmel
 from mwdust.util.tools import cos_sphere_dist
-from mwdust.DustMap3D import DustMap3D
+from mwdust.DustMap3D import DustMap3D, dust_dir, downloader
+
 _DEGTORAD= numpy.pi/180.
 class Drimmel03(DustMap3D):
  """extinction model from Drimmel et al. 2003 2003A&A...409..205D"""
@@ -262,7 +266,22 @@ def fit(self,l,b,dist,ext,e_ext):
  fs= amp*numpy.exp(pars[2])
  fo= amp*(1.-fd-fs)
  return (fd,fs,fo,numpy.exp(pars[3]))
-
+
+ @classmethod
+ def download(cls, test=False):
+ drimmel_folder_path = os.path.abspath(os.path.join(inspect.getfile(cls), "..", "util", "drimmeldata"))
+ drimmel_path = os.path.join(drimmel_folder_path, "data-for.tar.gz")
+ if not os.path.exists(drimmel_path):
+ if not os.path.exists(drimmel_folder_path):
+ os.mkdir(drimmel_folder_path)
+ _DRIMMEL_URL= "https://zenodo.org/record/7340108/files/data-for.tar.gz"
+ downloader(_DRIMMEL_URL, drimmel_path, "DRIMMEL03", test=test)
+ if not test:
+ file = tarfile.open(drimmel_path)
+ file.extractall(drimmel_folder_path)
+ file.close()
+
+
 def _fitFunc(pars,drim,l,b,dist,ext,e_ext):
  amp= numpy.exp(pars[0])
  fd= amp*numpy.exp(pars[1])
@@ -271,4 +290,3 @@ def _fitFunc(pars,drim,l,b,dist,ext,e_ext):
  dist_stretch= numpy.exp(pars[3])
  model_ext= drim(l,b,dist*dist_stretch,_fd=fd,_fs=fs,_fo=fo)
  return 0.5*numpy.sum((model_ext-ext)**2./e_ext**2.)
-

mwdust/DustMap3D.py

@@ -6,12 +6,7 @@
 ###############################################################################
 import numpy
 import os
-import inspect
-import urllib
-import gzip
-import tarfile
-import shutil
-from .util.tools import TqdmUpTo
+from .util.download import downloader
 try:
  from galpy.util import plot as bovy_plot
  _BOVY_PLOT_LOADED= True
@@ -40,7 +35,6 @@ def __init__(self,filter=None):
  """
  self._filter= filter
  self.download() # download the map
- return None
 
  def __call__(self,*args,**kwargs):
  """
@@ -107,152 +101,6 @@ def plot(self,l,b,*args,**kwargs):
  kwargs['ylabel']= r'$E(B-V)\,(\mathrm{mag})$'
  return bovy_plot.plot(ds,adust,*args,**kwargs)
 
-
- @staticmethod
- def __downloader(url, fullfilename, name, test=False):
- user_agent = "Mozilla/5.0"
- if test: # only for testing respond
- # import requests here to avoid including requests as a requirment in mwdust
- import requests
- # do not redirect, harvard dataverse redirect us to amazon cloud which won't respond to requests get get header
- # only making sure not 404, and site like harvard dataverse is directing us to somewhere
- r = requests.head(url, allow_redirects=False, verify=False)
- if r.status_code > 399: # allow HTTP code 2xx (Successful) and 3xx (Redirection) 
- raise ConnectionError(f"Cannot find {name} data file at {url}")
- r.close()
- else:
- with TqdmUpTo(unit="B", unit_scale=True, miniters=1, desc=name) as t:
- try:
- opener=urllib.request.build_opener()
- opener.addheaders=[("User-Agent", user_agent)]
- urllib.request.install_opener(opener)
- urllib.request.urlretrieve(url, fullfilename, reporthook=t.update_to)
- print(f"Downloaded {name} successfully to {fullfilename}")
- except urllib.error.HTTPError as emsg:
- if "404" in str(emsg):
- raise FileNotFoundError(f"{url} cannot be found on server, skipped")
- else:
- raise SystemError(f"Unknown error occurred - {emsg}")
-
  @classmethod
  def download(cls, test=False):
- """
- class method to download
- """
- if cls.__name__ == "SFD":
- sfd_ngp_path = os.path.join(dust_dir, "maps", "SFD_dust_4096_ngp.fits")
- if not os.path.exists(sfd_ngp_path):
- if not os.path.exists(os.path.join(dust_dir, "maps")):
- os.mkdir(os.path.join(dust_dir, "maps"))
- _SFD_URL_NGP= "https://svn.sdss.org/public/data/sdss/catalogs/dust/trunk/maps/SFD_dust_4096_ngp.fits"
- cls.__downloader(_SFD_URL_NGP, sfd_ngp_path, "SFD_NGP", test=test)
- sfd_sgp_path = os.path.join(dust_dir, "maps", "SFD_dust_4096_sgp.fits")
- if not os.path.exists(sfd_ngp_path):
- _SFD_URL_SGP= "https://svn.sdss.org/public/data/sdss/catalogs/dust/trunk/maps/SFD_dust_4096_sgp.fits"
- cls.__downloader(_SFD_URL_SGP, sfd_sgp_path, "SFD_SGP", test=test)
-
- elif cls.__name__ == "Drimmel03": 
- drimmel_folder_path = os.path.abspath(os.path.join(inspect.getfile(cls), "..", "util", "drimmeldata"))
- drimmel_path = os.path.join(drimmel_folder_path, "data-for.tar.gz")
- if not os.path.exists(drimmel_path):
- if not os.path.exists(drimmel_folder_path):
- os.mkdir(drimmel_folder_path)
- _DRIMMEL_URL= "https://zenodo.org/record/7340108/files/data-for.tar.gz"
- cls.__downloader(_DRIMMEL_URL, drimmel_path, "DRIMMEL03", test=test)
- if not test:
- file = tarfile.open(drimmel_path)
- file.extractall(drimmel_folder_path)
- file.close()
-
- elif cls.__name__ == "Marshall06":
- marshall_folder_path = os.path.join(dust_dir, "marshall06")
- marshall_path = os.path.join(marshall_folder_path, "table1.dat.gz")
- marshall_readme_path = os.path.join(dust_dir, "marshall06", "ReadMe")
- if not os.path.exists(marshall_path[:-3]):
- if not os.path.exists(marshall_folder_path):
- os.mkdir(marshall_folder_path)
- _MARSHALL_URL= "https://cdsarc.cds.unistra.fr/ftp/J/A+A/453/635/table1.dat.gz"
- cls.__downloader(_MARSHALL_URL, marshall_path, "MARSHALL06", test=test)
- if not test:
- with open(marshall_path, "rb") as inf, open(os.path.join(marshall_folder_path, "table1.dat"), "w", encoding="utf8") as tof:
- decom_str = gzip.decompress(inf.read()).decode("utf-8")
- tof.write(decom_str)
- if not os.path.exists(marshall_readme_path):
- _MARSHALL_README_URL= "https://cdsarc.cds.unistra.fr/ftp/J/A+A/453/635/ReadMe"
- cls.__downloader(_MARSHALL_README_URL, marshall_readme_path, "MARSHALL06 README", test=test)
-
- elif cls.__name__ == "Sale14":
- sale_folder_path = os.path.join(dust_dir, "sale14")
- sale_path = os.path.join(sale_folder_path, "Amap.tar.gz")
- if not os.path.exists(sale_path[:-6] + "dat"):
- if not os.path.exists(sale_folder_path):
- os.mkdir(sale_folder_path)
- _SALE_URL= "https://www.iphas.org/data/extinction/Amap.tar.gz"
- cls.__downloader(_SALE_URL, sale_path, "SALE14", test=test)
- if not test:
- file = tarfile.open(sale_path)
- file.extractall(sale_folder_path)
- file.close()
- # Fix one line in the dust map
- with open(os.path.join(sale_folder_path, "tmp.dat"), "w") as fout:
- with open(os.path.join(sale_folder_path, "Amap.dat"), "r") as fin:
- for line in fin:
- if "15960.40000" in line: # bad line
- newline= ''
- for ii,word in enumerate(line.split(' ')):
- if ii > 0: newline+= ' '
- if ii > 6 and len(word) > 9:
- word= "747.91400"
- newline+= word
- fout.write(newline+'\n')
- else:
- fout.write(line)
- shutil.move(os.path.join(sale_folder_path, "tmp.dat"), os.path.join(sale_folder_path, "Amap.dat"))
-
- elif cls.__name__ == "Green15":
- # Download Green et al. PanSTARRS data (alt.: https://dx.doi.org/10.7910/DVN/40C44C)
- green15_path = os.path.join(dust_dir, "green15", "dust-map-3d.h5")
- if not os.path.exists(green15_path):
- if not os.path.exists(os.path.join(dust_dir, "green15")):
- os.mkdir(os.path.join(dust_dir, "green15"))
- _GREEN15_URL = "https://faun.rc.fas.harvard.edu/pan1/ggreen/argonaut/data/dust-map-3d.h5"
- cls.__downloader(_GREEN15_URL, green15_path, "GREEN15", test=test)
-
- elif cls.__name__ == "Green17":
- # Download Green et al. 2018 PanSTARRS data
- green17_path = os.path.join(dust_dir, "green17", "bayestar2017.h5")
- if not os.path.exists(green17_path):
- if not os.path.exists(os.path.join(dust_dir, "green17")):
- os.mkdir(os.path.join(dust_dir, "green17"))
- _GREEN17_URL = "https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/LCYHJG/S7MP4P"
- cls.__downloader(_GREEN17_URL, green17_path, "GREEN17", test=test)
-
- elif cls.__name__ == "Green19":
- # Download Green et al. 2019 PanSTARRS data
- green19_path = os.path.join(dust_dir, "green19", "bayestar2019.h5")
- if not os.path.exists(green19_path):
- if not os.path.exists(os.path.join(dust_dir, "green19")):
- os.mkdir(os.path.join(dust_dir, "green19"))
- _GREEN19_URL = "https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/2EJ9TX/1CUGA1"
- cls.__downloader(_GREEN19_URL, green19_path, "GREEN19", test=test)
-
- elif cls.__name__ == "Combined19":
- # Download the combined map: Marshall+Green19+Drimmel for full sky coverage
- combined19_path = os.path.join(dust_dir, "combined19", "combine19.h5")
- if not os.path.exists(combined19_path):
- if not os.path.exists(os.path.join(dust_dir, "combined19")):
- os.mkdir(os.path.join(dust_dir, "combined19"))
- _COMBINED19_URL = "https://zenodo.org/record/3566060/files/combine19.h5"
- cls.__downloader(_COMBINED19_URL, combined19_path, "COMBINED19", test=test)
-
- elif cls.__name__ == "Combined15":
- # Download the combined map of Bovy et al. (2015): Marshall+Green+Drimmel for full sky coverage
- combined15_path = os.path.join(dust_dir, "combined15", "dust-map-3d.h5")
- if not os.path.exists(combined15_path):
- if not os.path.exists(os.path.join(dust_dir, "combined15")):
- os.mkdir(os.path.join(dust_dir, "combined15"))
- _COMBINED15_URL = "https://zenodo.org/record/31262/files/dust-map-3d.h5"
- cls.__downloader(_COMBINED15_URL, combined15_path, "COMBINED15", test=test)
-
- else:
- raise NameError(f"Unknown class: {cls.__name__}")
+ pass

mwdust/Green15.py

@@ -6,7 +6,7 @@
 import os, os.path
 import numpy
 import h5py
-from mwdust.DustMap3D import dust_dir
+from mwdust.DustMap3D import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
 _greendir= os.path.join(dust_dir, 'green15')
@@ -71,3 +71,12 @@ def substitute_sample(self,samplenum):
  dtype='object') #array to cache interpolated extinctions
  return None
 
+ @classmethod
+ def download(cls, test=False):
+ # Download Green et al. PanSTARRS data (alt.: https://dx.doi.org/10.7910/DVN/40C44C)
+ green15_path = os.path.join(dust_dir, "green15", "dust-map-3d.h5")
+ if not os.path.exists(green15_path):
+ if not os.path.exists(os.path.join(dust_dir, "green15")):
+ os.mkdir(os.path.join(dust_dir, "green15"))
+ _GREEN15_URL = "https://faun.rc.fas.harvard.edu/pan1/ggreen/argonaut/data/dust-map-3d.h5"
+ downloader(_GREEN15_URL, green15_path, "GREEN15", test=test)

mwdust/Green17.py

@@ -6,7 +6,7 @@
 import os, os.path
 import numpy
 import h5py
-from mwdust.DustMap3D import dust_dir
+from mwdust.DustMap3D import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
 _greendir= os.path.join(dust_dir, 'green17')
@@ -73,3 +73,12 @@ def substitute_sample(self,samplenum):
  dtype='object') #array to cache interpolated extinctions
  return None
 
+ @classmethod
+ def download(cls, test=False):
+ # Download Green et al. 2018 PanSTARRS data
+ green17_path = os.path.join(dust_dir, "green17", "bayestar2017.h5")
+ if not os.path.exists(green17_path):
+ if not os.path.exists(os.path.join(dust_dir, "green17")):
+ os.mkdir(os.path.join(dust_dir, "green17"))
+ _GREEN17_URL = "https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/LCYHJG/S7MP4P"
+ downloader(_GREEN17_URL, green17_path, "GREEN17", test=test)

mwdust/Green19.py

@@ -6,7 +6,7 @@
 import os, os.path
 import numpy
 import h5py
-from mwdust.DustMap3D import dust_dir
+from mwdust.DustMap3D import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
 _greendir= os.path.join(dust_dir, 'green19')
@@ -72,3 +72,12 @@ def substitute_sample(self,samplenum):
  dtype='object') #array to cache interpolated extinctions
  return None
 
+ @classmethod
+ def download(cls, test=False):
+ # Download Green et al. 2019 PanSTARRS data
+ green19_path = os.path.join(dust_dir, "green19", "bayestar2019.h5")
+ if not os.path.exists(green19_path):
+ if not os.path.exists(os.path.join(dust_dir, "green19")):
+ os.mkdir(os.path.join(dust_dir, "green19"))
+ _GREEN19_URL = "https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/2EJ9TX/1CUGA1"
+ downloader(_GREEN19_URL, green19_path, "GREEN19", test=test)

mwdust/Marshall06.py

@@ -5,12 +5,13 @@
 ###############################################################################
 import os, os.path
 import sys
+import gzip
 import numpy
 from scipy import interpolate
 from astropy.io import ascii
 from mwdust.util.extCurves import aebv
 from mwdust.util.tools import cos_sphere_dist
-from mwdust.DustMap3D import DustMap3D, dust_dir
+from mwdust.DustMap3D import DustMap3D, dust_dir, downloader
 try:
  from galpy.util import plot as bovy_plot
  _BOVY_PLOT_LOADED= True
@@ -240,3 +241,21 @@ def _lbIndx(self,l,b):
  lIndx= int(round((l+100.)/self._dl))
  bIndx= int(round((b+10.)/self._db))
  return lIndx*81+bIndx
+
+ @classmethod
+ def download(cls, test=False):
+ marshall_folder_path = os.path.join(dust_dir, "marshall06")
+ marshall_path = os.path.join(marshall_folder_path, "table1.dat.gz")
+ marshall_readme_path = os.path.join(dust_dir, "marshall06", "ReadMe")
+ if not os.path.exists(marshall_path[:-3]):
+ if not os.path.exists(marshall_folder_path):
+ os.mkdir(marshall_folder_path)
+ _MARSHALL_URL= "https://cdsarc.cds.unistra.fr/ftp/J/A+A/453/635/table1.dat.gz"
+ downloader(_MARSHALL_URL, marshall_path, "MARSHALL06", test=test)
+ if not test:
+ with open(marshall_path, "rb") as inf, open(os.path.join(marshall_folder_path, "table1.dat"), "w", encoding="utf8") as tof:
+ decom_str = gzip.decompress(inf.read()).decode("utf-8")
+ tof.write(decom_str)
+ if not os.path.exists(marshall_readme_path):
+ _MARSHALL_README_URL= "https://cdsarc.cds.unistra.fr/ftp/J/A+A/453/635/ReadMe"
+ downloader(_MARSHALL_README_URL, marshall_readme_path, "MARSHALL06 README", test=test)