Improve downloading of dust maps

.github/workflows/build.yml

@@ -40,17 +40,12 @@ jobs:
  if: runner.os != 'Windows'
  env:
  DUST_DIR: ${{github.workspace}}/data/
- run: mkdir -p $DUST_DIR 
- - name: Test downloads
- env:
- DUST_DIR: ${{github.workspace}}/data/
- run: |
- python setup.py install --test-downloads --all-downloads
+ run: mkdir -p $DUST_DIR
  - name: Install package
  env:
  DUST_DIR: ${{github.workspace}}/data/
  run: |
- python setup.py develop --no-downloads --download-sfd --download-green19 --verbose-downloads
+ python setup.py develop
  - name: Test package
  env:
  DUST_DIR: ${{github.workspace}}/data/

HISTORY.txt

@@ -1,3 +1,10 @@
+v1.2 (XXXX-XX-XX)
+==================
+
+- Re-jiggered the way dust maps are downloaded. They are now downloaded
+ upon first use of a dust map rather than at install time. This makes 
+ it possible to create installable wheels for mwdust.
+
 v1.2 (2023-02-21)
 ==================
 

README.rst

@@ -18,27 +18,16 @@ mwdust
 Installation
 -------------
 
-Please define an environment variable ``DUST_DIR`` before installing
-the code; this is a directory that will contain the dust data.
-
 Standard python setup.py build/install
 
 Either
 
-``sudo python setup.py install``
+``python setup.py install``
 
 or 
 
-``python setup.py install --prefix=/some/directory/``
-
-The installation automatically downloads the relevant dust data. You
-might have to define an environment variable ``SUDO_USER`` if not
-installing with sudo and you might have to use the ``-E`` option when
-you are installing with sudo to transfer your environment variables to
-sudo.
+``python setup.py install --user``
 
-Note that on Windows, you need to have the ``gzip`` utility 
-(e.g., by installing ``7zip``) to use the installation script. 
 Using custom implementations of necessary HEALPIx functions, basic 
 evaluation of extinction is available on all platforms (Linux, Mac OS,
 Windows) for all dust maps. However, some HEALPIx-based features like 
@@ -48,16 +37,22 @@ Install on Linux/Mac OS for full functionality.
 Dust Data
 ---------
 
-The code can automatically download all of the necessary data. By
-default, only the most commonly-used dust maps are downloaded; to
-download all maps, use the ``--all-downloads`` installation option
-(you can just re-run the installation with this option to add this
-later). The installation option ``--no-downloads`` turns all
-downloads off. By default, downloads are run without showing any 
-progress, but if you want to see the downloads's progression, use 
-``--verbose-downloads``.
+By default, dust maps are download when you use them for the first time. 
+If you define an environment variable ``DUST_DIR``, then all dust data
+downloaded by the code will be downloaded to this directory. If you do not
+set the ``DUST_DIR`` variable, then ``mwdust`` will download data to ``~/.mwdust``. 
+
+The code can download all of the necessary data at by running
+
+.. code-block:: python
+
+ from mwdust import download_all
+ download_all()
+
+Note that some of the maps are very large (multiple GB) and some of the downloads 
+are slow, so this may take a while.
 
-The data are put in subdirectories of a directory ``DUST_DIR``, with
+The data are put in subdirectories of a directory ``DUST_DIR`` or ``~/.mwdust``, with
 roughly the following lay-out::
 
  $DUST_DIR/
@@ -73,10 +68,10 @@ roughly the following lay-out::
  bayestar2019.h5
  maps/
  SFD_dust_4096_ngp.fits
- SFD_dust_4096_sgp.fits
+  SFD_dust_4096_sgp.fits
  marshall06/
  ReadMe
- table1.dat
+  table1.dat
  sale14/
  Amap.dat
  ReadMe

mwdust/Combined15.py

@@ -7,9 +7,10 @@
 import os, os.path
 import numpy
 import h5py
+from mwdust.util.download import downloader, dust_dir
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
-_combineddir= os.path.join(os.getenv('DUST_DIR'),'combined15')
+_combineddir= os.path.join(dust_dir, 'combined15')
 class Combined15(HierarchicalHealpixMap):
  """extinction model obtained from a combination of Marshall et al.
  (2006), Green et al. (2015), and Drimmel et al. (2003)"""
@@ -47,3 +48,14 @@ def __init__(self,filter=None,sf10=True,load_samples=False,
  dtype='object') #array to cache interpolated extinctions
  self._interpk= interpk
  return None
+
+ @classmethod
+ def download(cls, test=False):
+ # Download the combined map of Bovy et al. (2015): Marshall+Green+Drimmel for full sky coverage
+ combined15_path = os.path.join(dust_dir, "combined15", "dust-map-3d.h5")
+ if not os.path.exists(combined15_path):
+ if not os.path.exists(os.path.join(dust_dir, "combined15")):
+ os.mkdir(os.path.join(dust_dir, "combined15"))
+ _COMBINED15_URL = "https://zenodo.org/record/31262/files/dust-map-3d.h5"
+ downloader(_COMBINED15_URL, combined15_path, cls.__name__, test=test)
+ return None

mwdust/Combined19.py

@@ -7,9 +7,10 @@
 import os, os.path
 import numpy
 import h5py
+from mwdust.util.download import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
-_combineddir= os.path.join(os.getenv('DUST_DIR'),'combined19')
+_combineddir= os.path.join(dust_dir,'combined19')
 class Combined19(HierarchicalHealpixMap):
  """extinction model obtained from a combination of Marshall et al.
  (2006), Green et al. (2019), and Drimmel et al. (2003)"""
@@ -48,3 +49,14 @@ def __init__(self,filter=None,sf10=True,load_samples=False,
  dtype='object') #array to cache interpolated extinctions
  self._interpk= interpk
  return None
+
+ @classmethod
+ def download(cls, test=False):
+ # Download the combined map: Marshall+Green19+Drimmel for full sky coverage
+ combined19_path = os.path.join(dust_dir, "combined19", "combine19.h5")
+ if not os.path.exists(combined19_path):
+ if not os.path.exists(os.path.join(dust_dir, "combined19")):
+ os.mkdir(os.path.join(dust_dir, "combined19"))
+ _COMBINED19_URL = "https://zenodo.org/record/3566060/files/combine19.h5"
+ downloader(_COMBINED19_URL, combined19_path, cls.__name__, test=test)
+ return None

mwdust/Drimmel03.py

@@ -3,8 +3,11 @@
 # Drimmel03: extinction model from Drimmel et al. 2003 2003A&A...409..205D
 #
 ###############################################################################
+import os
 import copy
 import numpy
+import tarfile
+import inspect
 from scipy.ndimage import map_coordinates
 from scipy import optimize
 try:
@@ -13,7 +16,9 @@
 from mwdust.util.extCurves import aebv
 from mwdust.util import read_Drimmel
 from mwdust.util.tools import cos_sphere_dist
+from mwdust.util.download import dust_dir, downloader
 from mwdust.DustMap3D import DustMap3D
+
 _DEGTORAD= numpy.pi/180.
 class Drimmel03(DustMap3D):
  """extinction model from Drimmel et al. 2003 2003A&A...409..205D"""
@@ -262,7 +267,22 @@ def fit(self,l,b,dist,ext,e_ext):
  fs= amp*numpy.exp(pars[2])
  fo= amp*(1.-fd-fs)
  return (fd,fs,fo,numpy.exp(pars[3]))
-
+
+ @classmethod
+ def download(cls, test=False):
+ drimmel_folder_path = os.path.abspath(os.path.join(inspect.getfile(cls), "..", "util", "drimmeldata"))
+ drimmel_path = os.path.join(drimmel_folder_path, "data-for.tar.gz")
+ if not os.path.exists(drimmel_path):
+ if not os.path.exists(drimmel_folder_path):
+ os.mkdir(drimmel_folder_path)
+ _DRIMMEL_URL= "https://zenodo.org/record/7340108/files/data-for.tar.gz"
+ downloader(_DRIMMEL_URL, drimmel_path, cls.__name__, test=test)
+ if not test:
+ drim_file = tarfile.open(drimmel_path)
+ drim_file.extractall(drimmel_folder_path)
+ drim_file.close()
+ return None
+
 def _fitFunc(pars,drim,l,b,dist,ext,e_ext):
  amp= numpy.exp(pars[0])
  fd= amp*numpy.exp(pars[1])
@@ -271,4 +291,3 @@ def _fitFunc(pars,drim,l,b,dist,ext,e_ext):
  dist_stretch= numpy.exp(pars[3])
  model_ext= drim(l,b,dist*dist_stretch,_fd=fd,_fs=fs,_fo=fo)
  return 0.5*numpy.sum((model_ext-ext)**2./e_ext**2.)
-

mwdust/DustMap3D.py

@@ -10,6 +10,7 @@
  _BOVY_PLOT_LOADED= True
 except ImportError:
  _BOVY_PLOT_LOADED= False
+
 class DustMap3D:
  """top-level class for a 3D dust map; all other dust maps inherit from this"""
  def __init__(self,filter=None):
@@ -25,7 +26,7 @@ def __init__(self,filter=None):
  2013-11-24 - Started - Bovy (IAS)
  """
  self._filter= filter
- return None
+ self.download() # download the map
 
  def __call__(self,*args,**kwargs):
  """
@@ -47,7 +48,6 @@ def __call__(self,*args,**kwargs):
  try:
  return self._evaluate(l,b,d,**kwargs)
  except AttributeError:
- raise
  raise NotImplementedError("'_evaluate' for this DustMap3D not implemented yet")
 
  def plot(self,l,b,*args,**kwargs):
@@ -92,3 +92,7 @@ def plot(self,l,b,*args,**kwargs):
  else:
  kwargs['ylabel']= r'$E(B-V)\,(\mathrm{mag})$'
  return bovy_plot.plot(ds,adust,*args,**kwargs)
+
+ @classmethod
+ def download(cls, test=False):
+ pass

mwdust/Green15.py

@@ -6,9 +6,10 @@
 import os, os.path
 import numpy
 import h5py
+from mwdust.util.download import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
-_greendir= os.path.join(os.getenv('DUST_DIR'),'green15')
+_greendir= os.path.join(dust_dir, 'green15')
 class Green15(HierarchicalHealpixMap):
  """extinction model from Green et al. (2015)"""
  def __init__(self,filter=None,sf10=True,load_samples=False,
@@ -70,3 +71,13 @@ def substitute_sample(self,samplenum):
  dtype='object') #array to cache interpolated extinctions
  return None
 
+ @classmethod
+ def download(cls, test=False):
+ # Download Green et al. PanSTARRS data (alt.: https://dx.doi.org/10.7910/DVN/40C44C)
+ green15_path = os.path.join(dust_dir, "green15", "dust-map-3d.h5")
+ if not os.path.exists(green15_path):
+ if not os.path.exists(os.path.join(dust_dir, "green15")):
+ os.mkdir(os.path.join(dust_dir, "green15"))
+ _GREEN15_URL = "https://faun.rc.fas.harvard.edu/pan1/ggreen/argonaut/data/dust-map-3d.h5"
+ downloader(_GREEN15_URL, green15_path, cls.__name__, test=test)
+ return None

mwdust/Green17.py

@@ -6,9 +6,10 @@
 import os, os.path
 import numpy
 import h5py
+from mwdust.util.download import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
-_greendir= os.path.join(os.getenv('DUST_DIR'),'green17')
+_greendir= os.path.join(dust_dir, 'green17')
 class Green17(HierarchicalHealpixMap):
  """extinction model from Green et al. (2018)"""
  def __init__(self,filter=None,sf10=True,load_samples=False,
@@ -72,3 +73,13 @@ def substitute_sample(self,samplenum):
  dtype='object') #array to cache interpolated extinctions
  return None
 
+ @classmethod
+ def download(cls, test=False):
+ # Download Green et al. 2018 PanSTARRS data
+ green17_path = os.path.join(dust_dir, "green17", "bayestar2017.h5")
+ if not os.path.exists(green17_path):
+ if not os.path.exists(os.path.join(dust_dir, "green17")):
+ os.mkdir(os.path.join(dust_dir, "green17"))
+ _GREEN17_URL = "https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/LCYHJG/S7MP4P"
+ downloader(_GREEN17_URL, green17_path, cls.__name__, test=test)
+ return None

mwdust/Green19.py

@@ -6,9 +6,10 @@
 import os, os.path
 import numpy
 import h5py
+from mwdust.util.download import dust_dir, downloader
 from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
 _DEGTORAD= numpy.pi/180.
-_greendir= os.path.join(os.getenv('DUST_DIR'),'green19')
+_greendir= os.path.join(dust_dir, 'green19')
 class Green19(HierarchicalHealpixMap):
  """extinction model from Green et al. (2019)"""
  def __init__(self,filter=None,sf10=True,load_samples=False,
@@ -71,3 +72,13 @@ def substitute_sample(self,samplenum):
  dtype='object') #array to cache interpolated extinctions
  return None
 
+ @classmethod
+ def download(cls, test=False):
+ # Download Green et al. 2019 PanSTARRS data
+ green19_path = os.path.join(dust_dir, "green19", "bayestar2019.h5")
+ if not os.path.exists(green19_path):
+ if not os.path.exists(os.path.join(dust_dir, "green19")):
+ os.mkdir(os.path.join(dust_dir, "green19"))
+ _GREEN19_URL = "https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/2EJ9TX/1CUGA1"
+ downloader(_GREEN19_URL, green19_path, cls.__name__, test=test)
+ return None

mwdust/Marshall06.py

@@ -5,20 +5,23 @@
 ###############################################################################
 import os, os.path
 import sys
+import gzip
 import numpy
 from scipy import interpolate
 from astropy.io import ascii
 from mwdust.util.extCurves import aebv
 from mwdust.util.tools import cos_sphere_dist
+from mwdust.util.download import dust_dir, downloader
 from mwdust.DustMap3D import DustMap3D
+
 try:
  from galpy.util import plot as bovy_plot
  _BOVY_PLOT_LOADED= True
 except ImportError:
  _BOVY_PLOT_LOADED= False
 from matplotlib import pyplot
 _DEGTORAD= numpy.pi/180.
-_marshalldir= os.path.join(os.getenv('DUST_DIR'),'marshall06')
+_marshalldir= os.path.join(dust_dir,'marshall06')
 _ERASESTR= " "
 class Marshall06(DustMap3D):
  """extinction model from Marshall et al. 2006 2006A&A...453..635M"""
@@ -240,3 +243,23 @@ def _lbIndx(self,l,b):
  lIndx= int(round((l+100.)/self._dl))
  bIndx= int(round((b+10.)/self._db))
  return lIndx*81+bIndx
+
+ @classmethod
+ def download(cls, test=False):
+ marshall_folder_path = os.path.join(dust_dir, "marshall06")
+ marshall_path = os.path.join(marshall_folder_path, "table1.dat.gz")
+ marshall_readme_path = os.path.join(dust_dir, "marshall06", "ReadMe")
+ if not os.path.exists(marshall_path[:-3]):
+ if not os.path.exists(marshall_folder_path):
+ os.mkdir(marshall_folder_path)
+ _MARSHALL_URL= "https://cdsarc.cds.unistra.fr/ftp/J/A+A/453/635/table1.dat.gz"
+ downloader(_MARSHALL_URL, marshall_path, cls.__name__, test=test)
+ if not test:
+ with open(marshall_path, "rb") as inf, open(os.path.join(marshall_folder_path, "table1.dat"), "w", encoding="utf8") as tof:
+ decom_str = gzip.decompress(inf.read()).decode("utf-8")
+ tof.write(decom_str)
+ os.remove(marshall_path)
+ if not os.path.exists(marshall_readme_path):
+ _MARSHALL_README_URL= "https://cdsarc.cds.unistra.fr/ftp/J/A+A/453/635/ReadMe"
+ downloader(_MARSHALL_README_URL, marshall_readme_path, f"{cls.__name__} (ReadMe)", test=test)
+ return None