Make the following changes

 - felicette/sat_downloader.py, felicette/sat_processor.py: refactor handlers to make them more generic for different satellite data products, with new parameters. Integrate handling of sentinel-2 data.
 - felicette/utils/image_processing_utils.py: Make util functions more flexible to ignore 'rotation' in the case of sentinel
 - felicette/utils/file_manager.py: Handle downloading of sentinel data, with AWS credentials.
 - felicette/cli.py: Integrate new parameter '--product' to specify if sentinel or landsat data product is used. Change existing alias of --pan-enhancement to '-pan'.

felicette/cli.py

@@ -9,16 +9,17 @@
  download_landsat_data,
  search_satellite_data,
  preview_satellite_image,
+ download_data,
 )
 from felicette.utils.sys_utils import exit_cli, remove_dir
-from felicette.sat_processor import process_landsat_data
+from felicette.sat_processor import process_data
 
 
-def trigger_download_and_processing(landsat_item, bands):
+def trigger_download_and_processing(item, bands):
  # download data
- data_id = download_landsat_data(landsat_item, bands)
+ data_id = download_data(item, bands)
  # process data
- process_landsat_data(data_id, bands)
+ process_data(data_id, bands)
 
 
 @click.command()
@@ -31,7 +32,7 @@ def trigger_download_and_processing(landsat_item, bands):
 )
 @click.option("-l", "--location-name", type=str, help="Location name in string format")
 @click.option(
- "-p",
+ "-pan",
  "--pan-enhancement",
  default=False,
  is_flag=True,
@@ -57,7 +58,8 @@ def trigger_download_and_processing(landsat_item, bands):
  is_flag=True,
  help="Show the version number and quit",
 )
-def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, version):
+@click.option("-p", "--product", type=str, default="landsat", help="Product name 'landsat'/'sentinel'")
+def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, version, product):
  """Satellite imagery for dummies."""
  if version:
  version_no = pkg_resources.require("felicette")[0].version
@@ -68,37 +70,38 @@ def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, ve
  coordinates = geocoder_util(location_name)
 
  # unless specified, cloud_cover_lt is 10
- landsat_item = search_satellite_data(coordinates, 10)
+ item = search_satellite_data(coordinates, 10, product=product)
 
  # check if directory exists to save the data for this product id
- check_sat_path(landsat_item._data["id"])
+ check_sat_path(item._data["id"])
 
  # if preview option is set, download and preview image
  if not no_preview:
- preview_satellite_image(landsat_item)
+ preview_satellite_image(item)
 
  # set bands to process
  bands = [2, 3, 4]
- if pan_enhancement:
+ if pan_enhancement and (product != "sentinel"):
  bands.append(8)
 
  if vegetation:
  bands = [3, 4, 5]
 
  # NB: can't enable pan-enhancement with vegetation
+ # NB: can't enable pan-enhancement with sentinel
 
  try:
- trigger_download_and_processing(landsat_item, bands)
+ trigger_download_and_processing(item, bands)
  except RasterioIOError:
  response = input(
  "Local data for this location is corrupted, felicette will remove existing data to proceed, are you sure? [Y/n]"
  )
  if response in ["y", "Y", ""]:
  # remove file dir
- file_paths = file_paths_wrt_id(landsat_item._data["id"])
+ file_paths = file_paths_wrt_id(item._data["id"])
  remove_dir(file_paths["base"])
  # retry downloading and processing image with a clean directory
- trigger_download_and_processing(landsat_item, bands)
+ trigger_download_and_processing(item, bands)
  elif response in ["n", "N"]:
  exit_cli(print, "")
 

felicette/sat_downloader.py

@@ -1,6 +1,8 @@
 from satsearch import Search
 import sys
 from rich import print as rprint
+import requests
+import json
 
 from felicette.utils.geo_utils import get_tiny_bbox
 from felicette.utils.sys_utils import exit_cli
@@ -9,6 +11,7 @@
  save_to_file,
  data_file_exists,
  file_paths_wrt_id,
+ get_product_type_from_id,
 )
 
 
@@ -23,12 +26,18 @@ def handle_prompt_response(response):
  else:
  exit_cli(rprint, "[red]Sorry, invalid response. Exiting :([/red]")
 
-def search_satellite_data(coordinates, cloud_cover_lt, product="landsat-8-l1"):
+
+def search_satellite_data(coordinates, cloud_cover_lt, product="landsat"):
  """
  coordinates: bounding box's coordinates
  cloud_cover_lt: maximum cloud cover
- product: landsat-8-l1, sentinel-2-l1c
+ product: landsat, sentinel
  """
+ if product == "landsat":
+ product = "landsat-8-l1"
+ elif product == "sentinel":
+ product = "sentinel-2-l1c"
+
  search = Search(
  bbox=get_tiny_bbox(coordinates),
  query={
@@ -48,6 +57,7 @@ def search_satellite_data(coordinates, cloud_cover_lt, product="landsat-8-l1"):
  item = search_items[0]
  return item
 
+
 def preview_satellite_image(item):
  paths = file_paths_wrt_id(item._data["id"])
  # download image and save it in directory
@@ -70,6 +80,44 @@ def preview_satellite_image(item):
  return handle_prompt_response(response)
 
 
+def download_data(item, bands):
+ product_type = get_product_type_from_id(item._data["id"])
+ if product_type == "sentinel":
+ return download_sentinel_data(item, bands)
+ else:
+ return download_landsat_data(item, bands)
+
+
+def download_sentinel_data(item, bands):
+ # get paths w.r.t. id
+ paths = file_paths_wrt_id(item._data["id"])
+ # get meta info on path, to be used by boto3
+ info_response = requests.get(item.assets["info"]["href"])
+ info_response_json = json.loads(info_response.text)
+ # save bands generically
+ for band in bands:
+ # pass band id in metadata
+ info_response_json["band_id"] = band
+ band_filename = paths["b%s" % band]
+ if not data_file_exists(band_filename):
+ save_to_file(
+ item.assets["B0{}".format(band)]["href"],
+ band_filename,
+ item._data["id"],
+ "✗ required data doesn't exist, downloading %s %s"
+ % (band_tag_map["b" + str(band)], "band"),
+ meta=info_response_json,
+ )
+ else:
+ rprint(
+ "[green] ✓ ",
+ "required data exists for {} band".format(
+ band_tag_map["b" + str(band)]
+ ),
+ )
+ return item._data["id"]
+
+
 def download_landsat_data(landsat_item, bands):
 
  # get paths w.r.t. id

felicette/sat_processor.py

@@ -7,15 +7,15 @@
 
 from felicette.utils.color import color
 from felicette.utils.gdal_pansharpen import gdal_pansharpen
-from felicette.utils.file_manager import file_paths_wrt_id
+from felicette.utils.file_manager import file_paths_wrt_id, get_product_type_from_id
 from felicette.utils.image_processing_utils import process_sat_image
 from felicette.utils.sys_utils import display_file
 
 # increase PIL image processing pixels count limit
 PIL.Image.MAX_IMAGE_PIXELS = 933120000
 
 
-def process_landsat_vegetation(id, bands):
+def process_vegetation(id, bands, ops_string, angle_rotation=None):
 
  # get paths of files related to this id
  paths = file_paths_wrt_id(id)
@@ -53,7 +53,6 @@ def process_landsat_vegetation(id, bands):
 
  rprint("Let's make our 🌍 imagery a bit more colorful for a human eye!")
  # apply rio-color correction
- ops_string = "sigmoidal rgb 20 0.2"
  # refer to felicette.utils.color.py to see the parameters of this function
  # Bug: number of jobs if greater than 1, fails the job
  color(
@@ -68,7 +67,7 @@ def process_landsat_vegetation(id, bands):
  # resize and save as jpeg image
  print("Generated 🌍 images!🎉")
  rprint("[yellow]Please wait while I resize and crop the image :) [/yellow]")
- process_sat_image(paths["vegetation_path"], paths["vegetation_path_jpeg"])
+ process_sat_image(paths["vegetation_path"], paths["vegetation_path_jpeg"], rotate=angle_rotation)
  rprint("[blue]GeoTIFF saved at:[/blue]")
  print(paths["vegetation_path"])
  rprint("[blue]JPEG image saved at:[/blue]")
@@ -77,7 +76,7 @@ def process_landsat_vegetation(id, bands):
  display_file(paths["vegetation_path_jpeg"])
 
 
-def process_landsat_rgb(id, bands):
+def process_rgb(id, bands, ops_string, angle_rotation=None):
  # get paths of files related to this id
  paths = file_paths_wrt_id(id)
 
@@ -124,7 +123,6 @@ def process_landsat_rgb(id, bands):
 
  rprint("Let's make our 🌍 imagery a bit more colorful for a human eye!")
  # apply rio-color correction
- ops_string = "sigmoidal rgb 20 0.2"
  # refer to felicette.utils.color.py to see the parameters of this function
  # Bug: number of jobs if greater than 1, fails the job
  color(
@@ -139,7 +137,7 @@ def process_landsat_rgb(id, bands):
  # resize and save as jpeg image
  print("Generated 🌍 images!🎉")
  rprint("[yellow]Please wait while I resize and crop the image :) [/yellow]")
- process_sat_image(paths["output_path"], paths["output_path_jpeg"])
+ process_sat_image(paths["output_path"], paths["output_path_jpeg"], rotate=angle_rotation)
  rprint("[blue]GeoTIFF saved at:[/blue]")
  print(paths["output_path"])
  rprint("[blue]JPEG image saved at:[/blue]")
@@ -149,8 +147,23 @@ def process_landsat_rgb(id, bands):
 
 
 def process_landsat_data(id, bands):
-
+ ops_string = "sigmoidal rgb 20 0.2"
  if bands == [2, 3, 4] or bands == [2, 3, 4, 8]:
- process_landsat_rgb(id, bands)
+ process_rgb(id, bands, ops_string)
+ elif bands == [3, 4, 5]:
+ process_vegetation(id, bands, ops_string)
+
+def process_sentinel_data(id, bands):
+ ops_string = "gamma G 1.85 gamma B 1.85 gamma R 1.85 sigmoidal RGB 35 0.13 saturation 1.15"
+ angle_rotation = 0
+ if bands == [2, 3, 4]:
+ process_rgb(id, bands, ops_string, angle_rotation=angle_rotation)
  elif bands == [3, 4, 5]:
- process_landsat_vegetation(id, bands)
+ process_vegetation(id, bands, ops_string, angle_rotation=angle_rotation)
+
+def process_data(id, bands):
+ product_type = get_product_type_from_id(id)
+ if product_type == "sentinel":
+ process_sentinel_data(id, bands)
+ else:
+ process_landsat_data(id, bands)

felicette/utils/file_manager.py

@@ -3,8 +3,10 @@
 import requests
 from tqdm import tqdm
 from rich import print as rprint
+import boto3
 
 from felicette.constants import band_tag_map
+from felicette.utils.sys_utils import exit_cli
 
 workdir = os.path.join(os.path.expanduser("~"), "felicette-data")
 
@@ -16,44 +18,95 @@ def check_sat_path(id):
  os.makedirs(data_path, exist_ok=True)
 
 
-def save_to_file(url, filename, id, info_message):
+def hook(t):
+ def inner(bytes_amount):
+ t.update(bytes_amount)
+
+ return inner
+
+
+def save_to_file(url, filename, id, info_message, meta=None):
+ product_type = get_product_type_from_id(id)
  data_path = os.path.join(workdir, id)
  data_id = filename.split("/")[-1].split("-")[1].split(".")[0]
  rprint(info_message)
  file_path = os.path.join(data_path, filename)
- response = requests.get(url, stream=True)
- with tqdm.wrapattr(
- open(file_path, "wb"),
- "write",
- miniters=1,
- desc=data_id,
- total=int(response.headers.get("content-length", 0)),
- ) as fout:
- for chunk in response.iter_content(chunk_size=4096):
- fout.write(chunk)
- fout.close()
+
+ if product_type == "sentinel" and meta:
+ aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None)
+ aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", None)
+ # if access key or secret isn't defined, print error message and exit
+ if (not aws_access_key_id) or (not aws_secret_access_key):
+ exit_cli(rprint, "Error: [red]AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY[/red] must be set in environment variables to access Sentinel data.")
+ # prepare boto3 client
+ s3_client = boto3.Session().client(
+ "s3",
+ aws_access_key_id=aws_access_key_id,
+ aws_secret_access_key=aws_secret_access_key,
+ )
+ band = os.path.join(meta["path"], "B0%s.jp2" % (meta["band_id"]))
+ filesize = s3_client.head_object(
+ Bucket="sentinel-s2-l1c", Key=band, RequestPayer="requester"
+ ).get("ContentLength")
+ with tqdm(total=filesize, unit="B", unit_scale=True, desc=data_id) as t:
+ response = s3_client.download_file(
+ Bucket="sentinel-s2-l1c",
+ Key=band,
+ Filename=file_path,
+ ExtraArgs={"RequestPayer": "requester"},
+ Callback=hook(t),
+ )
+ else:
+ # for landsat, and preview images - resources which can be downloaded via http
+ response = requests.get(url, stream=True)
+ with tqdm.wrapattr(
+ open(file_path, "wb"),
+ "write",
+ miniters=1,
+ desc=data_id,
+ total=int(response.headers.get("content-length", 0)),
+ ) as fout:
+ for chunk in response.iter_content(chunk_size=4096):
+ fout.write(chunk)
+ fout.close()
 
 
 def data_file_exists(filename):
  return os.path.exists(filename)
 
 
+def get_product_type_from_id(id):
+ if "LC" in id:
+ return "landsat"
+ else:
+ return "sentinel"
+
+
 def file_paths_wrt_id(id):
  home_path_id = os.path.join(workdir, id)
+ extension = None
+ if get_product_type_from_id(id) == "landsat":
+ extension = "tiff"
+ else:
+ extension = "jp2"
  return {
  "base": home_path_id,
  "preview": os.path.join(home_path_id, "%s-preview.jpg" % (id)),
- "b5": os.path.join(home_path_id, "%s-b5.tiff" % (id)),
- "b4": os.path.join(home_path_id, "%s-b4.tiff" % (id)),
- "b3": os.path.join(home_path_id, "%s-b3.tiff" % (id)),
- "b2": os.path.join(home_path_id, "%s-b2.tiff" % (id)),
- "b8": os.path.join(home_path_id, "%s-b8.tiff" % (id)),
+ "b5": os.path.join(home_path_id, "%s-b5.%s" % (id, extension)),
+ "b4": os.path.join(home_path_id, "%s-b4.%s" % (id, extension)),
+ "b3": os.path.join(home_path_id, "%s-b3.%s" % (id, extension)),
+ "b2": os.path.join(home_path_id, "%s-b2.%s" % (id, extension)),
+ "b8": os.path.join(home_path_id, "%s-b8.%s" % (id, extension)),
  "stack": os.path.join(home_path_id, "%s-stack.tiff" % (id)),
  "pan_sharpened": os.path.join(home_path_id, "%s-pan.tiff" % (id)),
- "output_path": os.path.join(home_path_id, "%s-color-processed.tiff" % (id)),
+ "output_path": os.path.join(
+ home_path_id, "%s-color-processed.tiff" % (id)
+ ),
  "output_path_jpeg": os.path.join(
  home_path_id, "%s-color-processed.jpeg" % (id)
  ),
- "vegetation_path": os.path.join(home_path_id, "%s-vegetation.tiff" % (id)),
+ "vegetation_path": os.path.join(
+ home_path_id, "%s-vegetation.tiff" % (id)
+ ),
  "vegetation_path_jpeg": os.path.join(home_path_id, "%s-vegetation.jpeg" % (id)),
  }