-
Notifications
You must be signed in to change notification settings - Fork 20
/
download_data.py
45 lines (37 loc) · 1.39 KB
/
download_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import logging
import os.path
import shutil
from hyo2.abc.lib.ftp import Ftp
logging.basicConfig(level=logging.DEBUG,
format="%(levelname)-9s %(name)s.%(funcName)s:%(lineno)d > %(message)s")
logger = logging.getLogger(__name__)
clear_download_folder = False
# list of files to download
data_files = [
"rtofs_glo_3dz_f024_daily_3zsio.nc",
"rtofs_glo_3dz_f024_daily_3ztio.nc"
]
# create an empty `download` folder
download_folder = os.path.join(os.path.abspath(os.path.dirname(__file__)), "download")
if not os.path.exists(download_folder):
os.makedirs(download_folder)
elif clear_download_folder:
shutil.rmtree(download_folder)
os.makedirs(download_folder)
rtofs_folder = os.path.join(download_folder, "rtofs")
if not os.path.exists(rtofs_folder):
os.makedirs(rtofs_folder)
elif clear_download_folder:
shutil.rmtree(rtofs_folder)
os.makedirs(rtofs_folder)
# actually downloading the file with wget
for fid in data_files:
data_src = os.path.join("fromccom/hydroffice/smartmap", fid)
data_dst = os.path.abspath(os.path.join(rtofs_folder, fid))
print("> downloading %s to %s" % (data_src, data_dst))
try:
ftp = Ftp("ftp.ccom.unh.edu", show_progress=True, debug_mode=False)
ftp.get_file(data_src, data_dst, unzip_it=False)
ftp.disconnect()
except Exception as e:
logger.error('while downloading %s: %s' % (fid, e))