jemez_asf_api_download.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "beaf670f",
   "metadata": {},
   "source": [
    "# Jemez SnowEx UAVSAR Analysis\n",
    "\n",
    "This notebook directly downloads of UAVSAR data using the Alaska Satelitte Facility (ASF) API. It then converts the flat binary data provided by JPL to GeoTiffs. This work came out of the 2021 NASA SnowEx Hackweek\n",
    "\n",
    "Code By: Jack Tarricone, Paul Billecocq, and Zack Keskinen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "04e99033",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import re\n",
    "import zipfile\n",
    "import getpass\n",
    "from osgeo import gdal \n",
    "import os  # for chdir, getcwd, path.basename, path.exists\n",
    "import pandas as pd # for DatetimeIndex\n",
    "import codecs # for text parsing code\n",
    "import netrc\n",
    "import rasterio as rio\n",
    "from rasterio.plot import show # plotting raster data\n",
    "from rasterio.plot import show_hist #histograms of raster data\n",
    "import glob\n",
    "import requests\n",
    "from shapely.geometry import Polygon, mapping\n",
    "from datetime import datetime\n",
    "from subprocess import PIPE, Popen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b44601ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Enter Username:  jacktarricone\n",
      "Enter Password:  ············\n"
     ]
    }
   ],
   "source": [
    "# input NASA Earthdata credentials here\n",
    "ASF_USER = input(\"Enter Username: \")\n",
    "ASF_PASS = getpass.getpass(\"Enter Password: \")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c66bfae",
   "metadata": {},
   "source": [
    "### Function definitions to query ASF APi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e1c4a740",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_bbox_string(polygon):\n",
    "    '''\n",
    "    Builds the string to include in the ASF search request. The bbox consists of 4 comma-separated numbers: lower left longitude,latitude, and upper right longitude,latitude.\n",
    "    \n",
    "    Parmeters\n",
    "    ----------\n",
    "    polygon: shapely polygon\n",
    "    \n",
    "    Returns\n",
    "    ----------\n",
    "    polygon_string : string\n",
    "        String to include in the ASF request\n",
    "    '''\n",
    "    points = mapping(polygon)['coordinates'][0]\n",
    "    lower_left = points[0]\n",
    "    upper_right = points[2]\n",
    "    bbox_string = f'{lower_left[0]},{lower_left[1]},{upper_right[0]},{upper_right[1]}'\n",
    "        \n",
    "    return bbox_string\n",
    "\n",
    "def search_asf(platform, processingLevel, start, end, polygon, output_format):\n",
    "    '''\n",
    "    Search the ASF platform for images given the input parameters\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "        platform : string\n",
    "            Name of the imaging platform. Defaults to UAVSAR, but a list of supported platform is available on the ASF website\n",
    "        processingLevel : string\n",
    "            Processing level of the imaging product. \n",
    "            Possible values for UAVSAR : (KMZ, PROJECTED, PAULI, PROJECTED_ML5X5, STOKES, AMPLITUDE, BROWSE, \n",
    "            COMPLEX, DEM_TIFF, PROJECTED_ML3X3, METADATA, AMPLITUDE_GRD, INTERFEROMETRY, INTERFEROMETRY_GRD, THUMBNAIL)\n",
    "        start : datetime object\n",
    "            Start date of the search period.\n",
    "        end : datetime object\n",
    "            End date of the search period.\n",
    "        polygon : shapely polygon defining the Area of Interest,\n",
    "        output_format: string\n",
    "            Format being returned by the ASF API. Values : CSV, JSON, KML, METALINK, COUNT, DOWNLOAD, GEOJSON\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    Ouputs a search file\n",
    "    '''\n",
    "    base = 'https://api.daac.asf.alaska.edu/services/search/param'\n",
    "    start_date = start.strftime('%Y-%m-%dT%H:%M:%SUTC')\n",
    "    end_date = end.strftime('%Y-%m-%dT%H:%M:%SUTC')\n",
    "    aoi_string = build_bbox_string(polygon)\n",
    "    payload = {\n",
    "        'platform': platform,\n",
    "        'processingLevel': processingLevel,\n",
    "        'start': start,\n",
    "        'end': end,\n",
    "        'bbox': aoi_string,\n",
    "        'output': output_format\n",
    "    }\n",
    "    r = requests.get(base, params=payload)\n",
    "    \n",
    "    return r.json()\n",
    "\n",
    "def download_single_product(productUrl, path):\n",
    "    '''\n",
    "    Downloading function.\n",
    "    \n",
    "    Paramters\n",
    "    ----------\n",
    "    productUrl: string\n",
    "            Product download Url\n",
    "    path: string\n",
    "            path to download product to\n",
    "            \n",
    "    Returns\n",
    "    -------\n",
    "    Download the file\n",
    "    '''\n",
    "    process = Popen(['wget', productUrl, f'--user={ASF_USER}', f'--password={ASF_PASS}', '-P', path, '--progress=bar'], stderr=PIPE)\n",
    "    started = False\n",
    "    for line in process.stderr:\n",
    "        line = line.decode(\"utf-8\", \"replace\")\n",
    "        if started:\n",
    "            splited = line.split()\n",
    "            if len(splited) == 9:\n",
    "                percentage = splited[6]\n",
    "                speed = splited[7]\n",
    "                remaining = splited[8]\n",
    "                print(\"Downloaded {} with {} per second and {} left.\".format(percentage, speed, remaining), end='\\r')\n",
    "        elif line == os.linesep:\n",
    "            started = True\n",
    "    print(f\"\\nDone downloading {productUrl}\")\n",
    "    \n",
    "def search_and_bulk_download(platform, processingLevel, start, end, polygon, path):\n",
    "    '''\n",
    "    Search the ASF platform for images given the input parameters. Download found products\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "        platform : string\n",
    "            Name of the imaging platform. Defaults to UAVSAR, but a list of supported platform is available on the ASF website\n",
    "        processingLevel : string\n",
    "            Processing level of the imaging product. \n",
    "            Possible values for UAVSAR : (KMZ, PROJECTED, PAULI, PROJECTED_ML5X5, STOKES, AMPLITUDE, BROWSE, COMPLEX, DEM_TIFF, PROJECTED_ML3X3, METADATA, AMPLITUDE_GRD, INTERFEROMETRY, INTERFEROMETRY_GRD, THUMBNAIL)\n",
    "        start : datetime object\n",
    "            Start date of the search period.\n",
    "        end : datetime object\n",
    "            End date of the search period.\n",
    "        polygon : shapely polygon defining the Area of Interest,\n",
    "        path: string\n",
    "            Path to download product to\n",
    "        '''\n",
    "    results = search_asf(platform, processingLevel, start, end, polygon, output_format='JSON')[0]\n",
    "    print(f'{len(results)} product(s) found')\n",
    "    for result in results:\n",
    "        downloadUrl = result['downloadUrl']\n",
    "        download_single_product(downloadUrl, path)\n",
    "        print('Unzipping product')\n",
    "        with zipfile.ZipFile(path + result['fileName'], 'r') as zip_ref:\n",
    "            zip_ref.extractall(path + result['productName'])\n",
    "        print('Deleting original .zip')\n",
    "        os.remove(path + result['fileName'])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7dc6e755",
   "metadata": {},
   "outputs": [],
   "source": [
    "# folder is path to a folder with an .ann (or .txt) and .grd files (.amp1, .amp2, .cor, .unw, .int)\n",
    "\n",
    "def uavsar_tiff_convert(folder, verbose = False):\n",
    "    \"\"\"\n",
    "    Builds a header file for the input UAVSAR .grd file,\n",
    "    allowing the data to be read as a raster dataset.\n",
    "    :param folder:   the folder containing the UAVSAR .grd and .ann files\n",
    "    \"\"\"\n",
    "\n",
    "    os.chdir(folder)\n",
    "\n",
    "    # Empty lists to put information that will be recalled later.\n",
    "    Lines_list = []\n",
    "    Samples_list = []\n",
    "    Latitude_list = []\n",
    "    Longitude_list = []\n",
    "    Files_list = []\n",
    "\n",
    "    # Step 1: Look through folder and determine how many different flights there are\n",
    "    # by looking at the HDR files.\n",
    "    for files in os.listdir(folder):\n",
    "        if files [-4:] == \".grd\":\n",
    "            newfile = open(files[0:-4] + \".hdr\", 'w')\n",
    "            newfile.write(\"\"\"ENVI\n",
    "description = {DESCFIELD}\n",
    "samples = NSAMP\n",
    "lines = NLINE\n",
    "bands = 1\n",
    "header offset = 0\n",
    "data type = DATTYPE\n",
    "interleave = bsq\n",
    "sensor type = UAVSAR L-Band\n",
    "byte order = 0\n",
    "map info = {Geographic Lat/Lon, \n",
    "            1.000, \n",
    "            1.000, \n",
    "            LON, \n",
    "            LAT,  \n",
    "            0.0000555600000000, \n",
    "            0.0000555600000000, \n",
    "            WGS-84, units=Degrees}\n",
    "wavelength units = Unknown\n",
    "                \"\"\"\n",
    "                          )\n",
    "            newfile.close()\n",
    "            if files[0:18] not in Files_list:\n",
    "                Files_list.append(files[0:18])\n",
    "\n",
    "    #Variables used to recall indexed values.\n",
    "    var1 = 0\n",
    "\n",
    "    #Step 2: Look through the folder and locate the annotation file(s).\n",
    "    # These can be in either .txt or .ann file types.\n",
    "    for files in os.listdir(folder):\n",
    "        if Files_list[var1] and files[-4:] == \".txt\" or files[-4:] == \".ann\":\n",
    "            #Step 3: Once located, find the info we are interested in and append it to\n",
    "            # the appropriate list. We limit the variables to <=1 so that they only\n",
    "            # return two values (one for each polarization of\n",
    "            searchfile = codecs.open(files, encoding = 'windows-1252', errors='ignore')\n",
    "            for line in searchfile:\n",
    "                if \"Ground Range Data Latitude Lines\" in line:\n",
    "                    Lines = line[65:70]\n",
    "                    if verbose:\n",
    "                        print(f\"Number of Lines: {Lines}\")\n",
    "                    if Lines not in Lines_list:\n",
    "                        Lines_list.append(Lines)\n",
    "\n",
    "                elif \"Ground Range Data Longitude Samples\" in line:\n",
    "                    Samples = line[65:70]\n",
    "                    if verbose:\n",
    "                        print(f\"Number of Samples: {Samples}\")\n",
    "                    if Samples not in Samples_list:\n",
    "                        Samples_list.append(Samples)\n",
    "\n",
    "                elif \"Ground Range Data Starting Latitude\" in line:\n",
    "                    Latitude = line[65:85]\n",
    "                    if verbose:\n",
    "                        print(f\"Top left lat: {Latitude}\")\n",
    "                    if Latitude not in Latitude_list:\n",
    "                        Latitude_list.append(Latitude)\n",
    "\n",
    "                elif \"Ground Range Data Starting Longitude\" in line:\n",
    "                    Longitude = line[65:85]\n",
    "                    if verbose:\n",
    "                        print(f\"Top left Lon: {Longitude}\")\n",
    "                    if Longitude not in Longitude_list:\n",
    "                        Longitude_list.append(Longitude)\n",
    "    \n",
    "            #Reset the variables to zero for each different flight date.\n",
    "            var1 = 0\n",
    "            searchfile.close()\n",
    "\n",
    "\n",
    "    # Step 3: Open .hdr file and replace data for all type 4 (real numbers) data\n",
    "    # this all the .grd files expect for .int\n",
    "    for files in os.listdir(folder):\n",
    "        if files[-4:] == \".hdr\":\n",
    "            with open(files, \"r\") as sources:\n",
    "                lines = sources.readlines()\n",
    "            with open(files, \"w\") as sources:\n",
    "                for line in lines:\n",
    "                    if \"data type = DATTYPE\" in line:\n",
    "                        sources.write(re.sub(line[12:19], \"4\", line))\n",
    "                    elif \"DESCFIELD\" in line:\n",
    "                        sources.write(re.sub(line[15:24], folder, line))\n",
    "                    elif \"lines\" in line:\n",
    "                        sources.write(re.sub(line[8:13], Lines, line))\n",
    "                    elif \"samples\" in line:\n",
    "                        sources.write(re.sub(line[10:15], Samples, line))\n",
    "                    elif \"LAT\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Latitude, line))\n",
    "                    elif \"LON\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Longitude, line))\n",
    "                    else:\n",
    "                        sources.write(re.sub(line, line, line))\n",
    "    \n",
    "    # Step 3: Open .hdr file and replace data for .int file date type 6 (complex)                 \n",
    "    for files in os.listdir(folder):\n",
    "        if files[-8:] == \".int.hdr\":\n",
    "            with open(files, \"r\") as sources:\n",
    "                lines = sources.readlines()\n",
    "            with open(files, \"w\") as sources:\n",
    "                for line in lines:\n",
    "                    if \"data type = 4\" in line:\n",
    "                        sources.write(re.sub(line[12:13], \"6\", line))\n",
    "                    elif \"DESCFIELD\" in line:\n",
    "                        sources.write(re.sub(line[15:24], folder, line))\n",
    "                    elif \"lines\" in line:\n",
    "                        sources.write(re.sub(line[8:13], Lines, line))\n",
    "                    elif \"samples\" in line:\n",
    "                        sources.write(re.sub(line[10:15], Samples, line))\n",
    "                    elif \"LAT\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Latitude, line))\n",
    "                    elif \"LON\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Longitude, line))\n",
    "                    else:\n",
    "                        sources.write(re.sub(line, line, line))\n",
    "                        \n",
    "    \n",
    "    # Step 4: Now we have an .hdr file, the data is geocoded and can be loaded into python with rasterio\n",
    "    # once loaded in we use gdal.Translate to convert and save as a .tiff\n",
    "    \n",
    "    data_to_process = glob.glob(os.path.join(folder, '*.grd')) # list all .grd files\n",
    "    for data_path in data_to_process: # loop to open and translate .grd to .tiff, and save .tiffs using gdal\n",
    "        raster_dataset = gdal.Open(data_path, gdal.GA_ReadOnly)\n",
    "        raster = gdal.Translate(os.path.join(folder, os.path.basename(data_path) + '.tif'), raster_dataset, format = 'Gtiff', outputType = gdal.GDT_Float32)\n",
    "    \n",
    "    # Step 5: Save the .int raster, needs separate save because of the complex format\n",
    "    data_to_process = glob.glob(os.path.join(folder, '*.int.grd')) # list all .int.grd files (only 1)\n",
    "    for data_path in data_to_process:\n",
    "        raster_dataset = gdal.Open(data_path, gdal.GA_ReadOnly)\n",
    "        raster = gdal.Translate(os.path.join(folder, os.path.basename(data_path) + '.tif'), raster_dataset, format = 'Gtiff', outputType = gdal.GDT_CFloat32)\n",
    "        \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4459e757",
   "metadata": {},
   "outputs": [],
   "source": [
    "# jemez polygon for testing purposes (bounding box info from .ann)\n",
    "jemez_polygon = Polygon([(-106.7272, 35.7211),(-106.2372, 35.7211),(-106.2372, 36.0658),(-106.7272, 36.0658)])\n",
    "start_date = datetime.strptime('2020-02-19 11:00:00', '%Y-%m-%d %H:%M:%S') \n",
    "end_date = datetime.strptime('2020-02-26 11:00:00', '%Y-%m-%d %H:%M:%S') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bd735df4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# query API\n",
    "result = search_asf(platform='UAVSAR', processingLevel='INTERFEROMETRY_GRD', \n",
    "                    start=start_date, end=end_date, polygon=jemez_polygon, output_format='JSON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4ebf63dc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'absoluteOrbit': '-1',\n",
       " 'beamMode': 'RPI',\n",
       " 'beamModeType': 'RPI',\n",
       " 'beamSwath': None,\n",
       " 'browse': 'https://datapool.asf.alaska.edu/BROWSE/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090HH_01.cor.png',\n",
       " 'catSceneId': None,\n",
       " 'centerLat': '35.8884988092287',\n",
       " 'centerLon': '-106.439472086581',\n",
       " 'collectionName': 'Los Alamos, NM',\n",
       " 'configurationName': 'Repeat Pass Interferometry',\n",
       " 'doppler': '-1',\n",
       " 'downloadUrl': 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090_01_int_grd.zip',\n",
       " 'farEndLat': '36.04360921',\n",
       " 'farEndLon': '-106.33324373',\n",
       " 'farStartLat': '35.75907871',\n",
       " 'farStartLon': '-106.30749763',\n",
       " 'faradayRotation': None,\n",
       " 'fileName': 'alamos_35915_20008-000_20013-000_0007d_s01_L090_01_int_grd.zip',\n",
       " 'finalFrame': '718',\n",
       " 'firstFrame': '718',\n",
       " 'flightDirection': None,\n",
       " 'flightLine': '35915',\n",
       " 'formatName': None,\n",
       " 'frameNumber': '718',\n",
       " 'frequency': None,\n",
       " 'granuleName': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01',\n",
       " 'granuleType': 'UAVSAR_INSAR_SCENE',\n",
       " 'groupID': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01',\n",
       " 'incidenceAngle': None,\n",
       " 'insarGrouping': '-1',\n",
       " 'insarStackSize': '0',\n",
       " 'lookDirection': 'L',\n",
       " 'masterGranule': None,\n",
       " 'md5sum': 'c3f6fdf584f1d5026a93f47c9aa3ed75',\n",
       " 'missionName': 'Los Alamos, NM',\n",
       " 'nearEndLat': '36.02454445',\n",
       " 'nearEndLon': '-106.56254541',\n",
       " 'nearStartLat': '35.73394238',\n",
       " 'nearStartLon': '-106.55381725',\n",
       " 'offNadirAngle': '-1',\n",
       " 'percentCoherence': None,\n",
       " 'percentTroposphere': None,\n",
       " 'percentUnwrapped': None,\n",
       " 'platform': 'UAVSAR',\n",
       " 'pointingAngle': None,\n",
       " 'polarization': 'HH',\n",
       " 'processingDate': '2020-03-30T18:08:10Z',\n",
       " 'processingDescription': 'Ground projected interferogram product, one file per pair of repeat tracks, complex floating point format 8 bytes per pixel, little endian.',\n",
       " 'processingLevel': 'INTERFEROMETRY_GRD',\n",
       " 'processingType': 'L1',\n",
       " 'processingTypeDisplay': 'Ground Projected Interferogram',\n",
       " 'productName': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01',\n",
       " 'product_file_id': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01-INTERFEROMETRY_GRD',\n",
       " 'relativeOrbit': '35915',\n",
       " 'sarSceneId': None,\n",
       " 'sceneDate': '2020-02-26T16:27:24Z',\n",
       " 'sceneDateString': None,\n",
       " 'sceneId': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01',\n",
       " 'sensor': 'UAVSAR',\n",
       " 'sizeMB': '1212.75389767',\n",
       " 'slaveGranule': None,\n",
       " 'startTime': '2020-02-19T16:10:25Z',\n",
       " 'status': None,\n",
       " 'stopTime': '2020-02-26T16:27:24Z',\n",
       " 'stringFootprint': 'POLYGON((-106.562545 36.024544,-106.333244 36.043609,-106.307498 35.759079,-106.553817 35.733942,-106.562545 36.024544))',\n",
       " 'thumbnailUrl': 'https://datapool.asf.alaska.edu/THUMBNAIL/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090HH_01.amp1_THUMBNAIL.jpg',\n",
       " 'track': '35915',\n",
       " 'varianceTroposphere': None}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print first result\n",
    "result[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5601d9f9-a22c-44e2-ace5-319ec1ce6190",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'absoluteOrbit': '-1', 'beamMode': 'RPI', 'beamModeType': 'RPI', 'beamSwath': None, 'browse': 'https://datapool.asf.alaska.edu/BROWSE/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090HH_01.cor.png', 'catSceneId': None, 'centerLat': '35.8884988092287', 'centerLon': '-106.439472086581', 'collectionName': 'Los Alamos, NM', 'configurationName': 'Repeat Pass Interferometry', 'doppler': '-1', 'downloadUrl': 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090_01_int_grd.zip', 'farEndLat': '36.04360921', 'farEndLon': '-106.33324373', 'farStartLat': '35.75907871', 'farStartLon': '-106.30749763', 'faradayRotation': None, 'fileName': 'alamos_35915_20008-000_20013-000_0007d_s01_L090_01_int_grd.zip', 'finalFrame': '718', 'firstFrame': '718', 'flightDirection': None, 'flightLine': '35915', 'formatName': None, 'frameNumber': '718', 'frequency': None, 'granuleName': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01', 'granuleType': 'UAVSAR_INSAR_SCENE', 'groupID': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01', 'incidenceAngle': None, 'insarGrouping': '-1', 'insarStackSize': '0', 'lookDirection': 'L', 'masterGranule': None, 'md5sum': 'c3f6fdf584f1d5026a93f47c9aa3ed75', 'missionName': 'Los Alamos, NM', 'nearEndLat': '36.02454445', 'nearEndLon': '-106.56254541', 'nearStartLat': '35.73394238', 'nearStartLon': '-106.55381725', 'offNadirAngle': '-1', 'percentCoherence': None, 'percentTroposphere': None, 'percentUnwrapped': None, 'platform': 'UAVSAR', 'pointingAngle': None, 'polarization': 'HH', 'processingDate': '2020-03-30T18:08:10Z', 'processingDescription': 'Ground projected interferogram product, one file per pair of repeat tracks, complex floating point format 8 bytes per pixel, little endian.', 'processingLevel': 'INTERFEROMETRY_GRD', 'processingType': 'L1', 'processingTypeDisplay': 'Ground Projected Interferogram', 'productName': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01', 'product_file_id': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01-INTERFEROMETRY_GRD', 'relativeOrbit': '35915', 'sarSceneId': None, 'sceneDate': '2020-02-26T16:27:24Z', 'sceneDateString': None, 'sceneId': 'UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_01', 'sensor': 'UAVSAR', 'sizeMB': '1212.75389767', 'slaveGranule': None, 'startTime': '2020-02-19T16:10:25Z', 'status': None, 'stopTime': '2020-02-26T16:27:24Z', 'stringFootprint': 'POLYGON((-106.562545 36.024544,-106.333244 36.043609,-106.307498 35.759079,-106.553817 35.733942,-106.562545 36.024544))', 'thumbnailUrl': 'https://datapool.asf.alaska.edu/THUMBNAIL/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090HH_01.amp1_THUMBNAIL.jpg', 'track': '35915', 'varianceTroposphere': None}\n"
     ]
    }
   ],
   "source": [
    "print(result[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f0991a3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 product(s) found\n",
      "Downloaded 99% with 15.6M per second and 0s left....\n",
      "Done downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090_01_int_grd.zip\n",
      "Unzipping product\n",
      "Deleting original .zip\n",
      "Downloaded 99% with 13.8M per second and 0s left....\n",
      "Done downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/alamos_35915_20008-000_20013-000_0007d_s01_L090_02_int_grd.zip\n",
      "Unzipping product\n",
      "Deleting original .zip\n",
      "Downloaded 99% with 11.1M per second and 0s left....\n",
      "Done downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/alamos_35915_20005-003_20008-000_0007d_s01_L090_01_int_grd.zip\n",
      "Unzipping product\n",
      "Deleting original .zip\n"
     ]
    }
   ],
   "source": [
    "# UAVSAR InSAR files are pretty large, so this may take a bit of time\n",
    "search_and_bulk_download(platform='UAVSAR', processingLevel='INTERFEROMETRY_GRD', \n",
    "                         start=start_date, end=end_date, \n",
    "                         polygon=jemez_polygon, path='/Users/jacktarricone/jemez_data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "30d1f3ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/jacktarricone/jemez_data/UA_alamos_35915_20008-000_20013-000_0007d_s01_L090_02\n"
     ]
    }
   ],
   "source": [
    "# get new path for folder of insar data just downloaded\n",
    "new_path_list = glob.glob('/Users/jacktarricone/jemez_data/*')\n",
    "new_path = new_path_list[1] # select first list elemet\n",
    "print(new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "51cde922",
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert files to geotiffs and save in same folder\n",
    "uavsar_tiff_convert(new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6d2a814d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove unwanted files\n",
    "os.chdir(new_path)\n",
    "grd = glob.glob('*.grd') #define .grd\n",
    "hdr = glob.glob('*.hdr*') #define .hdr\n",
    "int_file = glob.glob('*.int*') #define .int\n",
    "\n",
    "# remove both\n",
    "for f in grd:\n",
    "    os.remove(f)\n",
    "    \n",
    "for f in hdr:\n",
    "    os.remove(f)\n",
    "    \n",
    "for f in int_file:\n",
    "    os.remove(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35cf438b-21d9-4e9b-9b23-9b976dbabd28",
   "metadata": {},
   "source": [
    "## Define only the HH files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3102351b-32b1-4e91-9a48-b8efac1cbb65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "alamos_35915_20005-003_20008-000_0007d_s01_L090HH_01.cor.grd.tif\n",
      "alamos_35915_20005-003_20008-000_0007d_s01_L090HH_01.unw.grd.tif\n",
      "alamos_35915_20005-003_20008-000_0007d_s01_L090HH_01.hgt.grd.tif\n"
     ]
    }
   ],
   "source": [
    "# coherence\n",
    "for cor in glob.glob(\"*HH*cor.grd.tif\"):\n",
    "    print(cor)\n",
    "\n",
    "# unwrapped phase\n",
    "for unw in glob.glob(\"*HH*unw.grd.tif\"):\n",
    "    print(unw)\n",
    "\n",
    "# dem used in processing\n",
    "for dem in glob.glob(\"*HH*hgt.grd.tif\"):\n",
    "    print(dem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "309ed33d-b822-4431-bc72-61d0b9de26a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# open using rio\n",
    "cor_rast = rio.open(cor)\n",
    "unw_rast = rio.open(unw)\n",
    "dem_rast = rio.open(dem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e48a441b-c7e5-4b14-9bce-33ceec1b5f2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# jemez polygon for testing purposes (bounding box info from .ann)\n",
    "jemez_polygon = Polygon([(-106.7272, 35.7211),(-106.2372, 35.7211),(-106.2372, 36.0658),(-106.7272, 36.0658)])\n",
    "start_date = datetime.strptime('2020-02-12 11:00:00', '%Y-%m-%d %H:%M:%S') \n",
    "end_date = datetime.strptime('2020-02-18 11:00:00', '%Y-%m-%d %H:%M:%S') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fda6ead7-b9a9-46e4-847e-c2f641fe7875",
   "metadata": {},
   "outputs": [],
   "source": [
    "# query API\n",
    "result2 = search_asf(platform='UAVSAR', processingLevel='AMPLITUDE_GRD', \n",
    "                    start=start_date, end=end_date, polygon=jemez_polygon, output_format='JSON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a0c06042-163c-4ce1-8418-7ade2abc25f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 product(s) found\n",
      "Downloaded 99% with 13.2M per second and 0s left..t.\n",
      "Done downloading https://datapool.asf.alaska.edu/AMPLITUDE_GRD/UA/alamos_35915_20005-003_20008-000_0007d_s01_L090_01_amp_grd.zip\n",
      "Unzipping product\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/Users/jacktarricone/ch1_jemez_data/feb12-19alamos_35915_20005-003_20008-000_0007d_s01_L090_01_amp_grd.zip'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/7d/61kbxptj5yl29nbrh216cv_40000gn/T/ipykernel_53053/3346933013.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# UAVSAR InSAR files are pretty large, so this may take a bit of time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m search_and_bulk_download(platform='UAVSAR', processingLevel='AMPLITUDE_GRD', \n\u001b[0m\u001b[1;32m      3\u001b[0m                          \u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstart_date\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mend_date\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                          polygon=jemez_polygon, path='/Users/jacktarricone/ch1_jemez_data/feb12-19')\n",
      "\u001b[0;32m/var/folders/7d/61kbxptj5yl29nbrh216cv_40000gn/T/ipykernel_53053/1775541095.py\u001b[0m in \u001b[0;36msearch_and_bulk_download\u001b[0;34m(platform, processingLevel, start, end, polygon, path)\u001b[0m\n\u001b[1;32m    113\u001b[0m         \u001b[0mdownload_single_product\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdownloadUrl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Unzipping product'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mzipfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mZipFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fileName'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mzip_ref\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    116\u001b[0m             \u001b[0mzip_ref\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'productName'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Deleting original .zip'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/test_env/lib/python3.8/zipfile.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, file, mode, compression, allowZip64, compresslevel, strict_timestamps)\u001b[0m\n\u001b[1;32m   1249\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1250\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1251\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilemode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1252\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1253\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mfilemode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmodeDict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/Users/jacktarricone/ch1_jemez_data/feb12-19alamos_35915_20005-003_20008-000_0007d_s01_L090_01_amp_grd.zip'"
     ]
    }
   ],
   "source": [
    "# UAVSAR InSAR files are pretty large, so this may take a bit of time\n",
    "search_and_bulk_download(platform='UAVSAR', processingLevel='AMPLITUDE_GRD', \n",
    "                         start=start_date, end=end_date, \n",
    "                         polygon=jemez_polygon, path='/Users/jacktarricone/ch1_jemez_data/feb12-19/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3e0b395-20fd-4c4f-b178-0315ddbdafe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get new path for folder of insar data just downloaded\n",
    "amp_list = glob.glob('/Users/jacktarricone/jemez_data/*amp*')\n",
    "amp_path = amp_list[1] # select first list elemet\n",
    "print(new_path)\n",
    "\n",
    "# convert files to geotiffs and save in same folder\n",
    "uavsar_tiff_convert(new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bd12c2ac-703d-4f0e-86ad-ecb42ac7c8b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "uavsar_tiff_convert('/Users/jacktarricone/ch1_jemez_data/feb12-19/alamos_35915_20005-003_20008-000_0007d_s01_L090_01_amp_grd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a96b6a21-c3ca-4296-9842-9bce378e25d9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:test_env]",
   "language": "python",
   "name": "conda-env-test_env-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}