Extract STAC Items_git.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "404b833d-b466-44e9-9c62-0582bb1e9e5c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 39 items matching bbox and datetime criteria for 20180108T184332_dmi_prep.nc.\n",
      "Filtered down to 20 EW mode and LEVEL1 items for 20180108T184332_dmi_prep.nc.\n",
      "Found 43 items matching bbox and datetime criteria for 20180306T101622_dmi_prep.nc.\n",
      "Filtered down to 18 EW mode and LEVEL1 items for 20180306T101622_dmi_prep.nc.\n",
      "Found 6 items matching bbox and datetime criteria for 20180311T111524_cis_prep.nc.\n",
      "Filtered down to 4 EW mode and LEVEL1 items for 20180311T111524_cis_prep.nc.\n",
      "Found 32 items matching bbox and datetime criteria for 20180325T194759_dmi_prep.nc.\n",
      "Filtered down to 10 EW mode and LEVEL1 items for 20180325T194759_dmi_prep.nc.\n",
      "Found 9 items matching bbox and datetime criteria for 20180707T122209_cis_prep.nc.\n",
      "Filtered down to 6 EW mode and LEVEL1 items for 20180707T122209_cis_prep.nc.\n",
      "Found 32 items matching bbox and datetime criteria for 20180721T120156_cis_prep.nc.\n",
      "Filtered down to 24 EW mode and LEVEL1 items for 20180721T120156_cis_prep.nc.\n",
      "Found 24 items matching bbox and datetime criteria for 20180829T113858_cis_prep.nc.\n",
      "Filtered down to 8 EW mode and LEVEL1 items for 20180829T113858_cis_prep.nc.\n",
      "Found 29 items matching bbox and datetime criteria for 20180902T124300_cis_prep.nc.\n",
      "Filtered down to 24 EW mode and LEVEL1 items for 20180902T124300_cis_prep.nc.\n",
      "Found 30 items matching bbox and datetime criteria for 20180905T130632_cis_prep.nc.\n",
      "Filtered down to 24 EW mode and LEVEL1 items for 20180905T130632_cis_prep.nc.\n",
      "Found 56 items matching bbox and datetime criteria for 20180914T115342_cis_prep.nc.\n",
      "Filtered down to 47 EW mode and LEVEL1 items for 20180914T115342_cis_prep.nc.\n",
      "Found 43 items matching bbox and datetime criteria for 20190607T082058_dmi_prep.nc.\n",
      "Filtered down to 32 EW mode and LEVEL1 items for 20190607T082058_dmi_prep.nc.\n",
      "Found 51 items matching bbox and datetime criteria for 20190619T185115_dmi_prep.nc.\n",
      "Filtered down to 30 EW mode and LEVEL1 items for 20190619T185115_dmi_prep.nc.\n",
      "Found 49 items matching bbox and datetime criteria for 20190808T120918_cis_prep.nc.\n",
      "Filtered down to 36 EW mode and LEVEL1 items for 20190808T120918_cis_prep.nc.\n",
      "Found 32 items matching bbox and datetime criteria for 20190812T131548_cis_prep.nc.\n",
      "Filtered down to 14 EW mode and LEVEL1 items for 20190812T131548_cis_prep.nc.\n",
      "Found 21 items matching bbox and datetime criteria for 20190819T153633_cis_prep.nc.\n",
      "Filtered down to 14 EW mode and LEVEL1 items for 20190819T153633_cis_prep.nc.\n",
      "Found 32 items matching bbox and datetime criteria for 20190831T144552_cis_prep.nc.\n",
      "Filtered down to 10 EW mode and LEVEL1 items for 20190831T144552_cis_prep.nc.\n",
      "Found 32 items matching bbox and datetime criteria for 20190917T131550_cis_prep.nc.\n",
      "Filtered down to 14 EW mode and LEVEL1 items for 20190917T131550_cis_prep.nc.\n",
      "Found 33 items matching bbox and datetime criteria for 20200423T214037_cis_prep.nc.\n",
      "Filtered down to 16 EW mode and LEVEL1 items for 20200423T214037_cis_prep.nc.\n",
      "Found 33 items matching bbox and datetime criteria for 20200619T122818_cis_prep.nc.\n",
      "Filtered down to 16 EW mode and LEVEL1 items for 20200619T122818_cis_prep.nc.\n",
      "Found 63 items matching bbox and datetime criteria for 20200728T120112_cis_prep.nc.\n",
      "Filtered down to 54 EW mode and LEVEL1 items for 20200728T120112_cis_prep.nc.\n",
      "Found 51 items matching bbox and datetime criteria for 20200819T104119_cis_prep.nc.\n",
      "Filtered down to 42 EW mode and LEVEL1 items for 20200819T104119_cis_prep.nc.\n",
      "Found 42 items matching bbox and datetime criteria for 20200910T194821_dmi_prep.nc.\n",
      "Filtered down to 20 EW mode and LEVEL1 items for 20200910T194821_dmi_prep.nc.\n",
      "Found 24 items matching bbox and datetime criteria for 20200922T172924_cis_prep.nc.\n",
      "Filtered down to 16 EW mode and LEVEL1 items for 20200922T172924_cis_prep.nc.\n",
      "Found 48 items matching bbox and datetime criteria for 20201022T194736_dmi_prep.nc.\n",
      "Filtered down to 20 EW mode and LEVEL1 items for 20201022T194736_dmi_prep.nc.\n",
      "Found 22 items matching bbox and datetime criteria for 20201104T123050_cis_prep.nc.\n",
      "Filtered down to 8 EW mode and LEVEL1 items for 20201104T123050_cis_prep.nc.\n",
      "Total matching items collected: 25\n",
      "S1A_EW_GRDM_1SDH_20180108T184332_20180108T184437_020067_022330_93EB.SAFE\n",
      "S1A_EW_GRDM_1SDH_20180306T101622_20180306T101722_020893_023D76_206D.SAFE\n",
      "S1B_EW_GRDM_1SDH_20180311T111524_20180311T111623_009983_01218C_F0B7.SAFE\n",
      "S1A_EW_GRDM_1SDH_20180325T194759_20180325T194859_021176_024676_449C.SAFE\n",
      "S1A_EW_GRDM_1SDH_20180707T122209_20180707T122309_022688_027552_A48E.SAFE\n",
      "S1A_EW_GRDM_1SDH_20180721T120156_20180721T120256_022892_027B98_56D9.SAFE\n",
      "S1B_EW_GRDM_1SDH_20180829T113858_20180829T114003_012477_017029_869A.SAFE\n",
      "S1B_EW_GRDM_1SDH_20180902T124300_20180902T124400_012536_017204_4133.SAFE\n",
      "S1B_EW_GRDM_1SDH_20180905T130632_20180905T130732_012580_017363_A7E8.SAFE\n",
      "S1A_EW_GRDM_1SDH_20180914T115342_20180914T115442_023694_02952F_CA97.SAFE\n",
      "S1A_EW_GRDM_1SDH_20190607T082058_20190607T082158_027571_031C84_37CA.SAFE\n",
      "S1B_EW_GRDM_1SDH_20190619T185115_20190619T185219_016769_01F8F8_AB64.SAFE\n",
      "S1B_EW_GRDM_1SDH_20190808T120918_20190808T121018_017494_020E65_AA35.SAFE\n",
      "S1B_EW_GRDM_1SDH_20190812T131548_20190812T131648_017553_021035_836B.SAFE\n",
      "S1A_EW_GRDM_1SDH_20190819T153633_20190819T153733_028640_033D98_D165.SAFE\n",
      "S1B_EW_GRDM_1SDH_20190831T144552_20190831T144652_017831_0218EB_1D2C.SAFE\n",
      "S1B_EW_GRDM_1SDH_20190917T131550_20190917T131650_018078_022090_69A9.SAFE\n",
      "S1B_EW_GRDM_1SDH_20200423T214037_20200423T214142_021277_02862A_4B23.SAFE\n",
      "S1A_EW_GRDM_1SDH_20200619T122818_20200619T122923_033086_03D537_EA36.SAFE\n",
      "S1B_EW_GRDM_1SDH_20200728T120112_20200728T120212_022671_02B07F_57F8.SAFE\n",
      "S1B_EW_GRDM_1SDH_20200819T104119_20200819T104219_022991_02BA50_1805.SAFE\n",
      "S1A_EW_GRDM_1SDH_20200910T194821_20200910T194921_034301_03FCB0_E46C.SAFE\n",
      "S1B_EW_GRDM_1SDH_20200922T172924_20200922T173024_023491_02C9FB_0F67.SAFE\n",
      "S1B_EW_GRDM_1SDH_20201022T194736_20201022T194836_023930_02D7BE_BFD6.SAFE\n",
      "S1B_EW_GRDM_1SDH_20201104T123050_20201104T123132_024115_02DD70_0B82.SAFE\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import xarray as xr\n",
    "from pystac_client import Client\n",
    "\n",
    "# Path to the directory containing NetCDF files\n",
    "netcdf_directory = \"C:/Users/r134p694_a/Training Netcdf\"\n",
    "\n",
    "# STAC API client setup\n",
    "client = Client.open(\"https://catalogue.dataspace.copernicus.eu/stac\") \n",
    "collection = \"SENTINEL-1\"\n",
    "\n",
    "# Initialize an empty list to collect matching filtered items\n",
    "matching_items = []\n",
    "\n",
    "# Iterate through each NetCDF file in the directory\n",
    "for netcdf_file in os.listdir(netcdf_directory):\n",
    "    if netcdf_file.endswith('.nc'):\n",
    "        # Read the NetCDF file\n",
    "        ds = xr.open_dataset(os.path.join(netcdf_directory, netcdf_file))\n",
    "        \n",
    "        # Calculate the bounding box\n",
    "        lat_min = np.min(ds.sar_grid2d_latitude.values)\n",
    "        lat_max = np.max(ds.sar_grid2d_latitude.values)\n",
    "        lon_min = np.min(ds.sar_grid2d_longitude.values)\n",
    "        lon_max = np.max(ds.sar_grid2d_longitude.values)\n",
    "\n",
    "        # Define the bounding box (min_lon, min_lat, max_lon, max_lat)\n",
    "        bbox = [lon_min, lat_min, lon_max, lat_max]\n",
    "\n",
    "        # Extract the date from the scene ID in attributes (if exists)\n",
    "        scene_id = ds.attrs.get('scene_id', '')  # Adjust based on actual attribute name\n",
    "        date = scene_id[:8] if scene_id else ''  # Extract date part\n",
    "\n",
    "        # Perform the Search with bbox and datetime\n",
    "        search = client.search(\n",
    "            collections=[collection],\n",
    "            bbox=bbox,\n",
    "            datetime=f\"{date[:4]}-{date[4:6]}-{date[6:8]}\",\n",
    "            limit=100\n",
    "        )\n",
    "\n",
    "        # Fetch All Matching Items\n",
    "        try:\n",
    "            items = list(search.get_items())\n",
    "            print(f\"Found {len(items)} items matching bbox and datetime criteria for {netcdf_file}.\")\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred during the search: {e}\")\n",
    "            continue\n",
    "\n",
    "        # Initialize an empty list to store filtered items\n",
    "        filtered_items = []\n",
    "\n",
    "        # Iterate through retrieved items and filter by 's1:instrument_mode' and 'processingLevel'\n",
    "        for item in items:\n",
    "            instrument_mode = item.properties.get('operationalMode', '')\n",
    "            processing_level = item.properties.get('processingLevel', '')\n",
    "            \n",
    "            # Check if both conditions are met\n",
    "            if instrument_mode == 'EW' and processing_level == 'LEVEL1':\n",
    "                filtered_items.append(item)\n",
    "\n",
    "        print(f\"Filtered down to {len(filtered_items)} EW mode and LEVEL1 items for {netcdf_file}.\")\n",
    "\n",
    "        # Get the original ID from the dataset attributes\n",
    "        original_id = ds.attrs.get('original_id', '')  # Adjust based on actual attribute name\n",
    "        # Extract the part of original_id before '_icechart'\n",
    "        original_id_prefix = original_id.split('_icechart')[0] if '_icechart' in original_id else original_id\n",
    "\n",
    "        # Match the original_id with filtered items\n",
    "        for filtered_item in filtered_items:\n",
    "            filtered_id = filtered_item.id.split('.')[0]  # Remove the \".SAFE\" part\n",
    "            if original_id_prefix in filtered_id:  # Check for a match\n",
    "                matching_items.append(filtered_item)\n",
    "                break  # Stop after finding the first match\n",
    "\n",
    "# Output the matched items\n",
    "print(f\"Total matching items collected: {len(matching_items)}\")\n",
    "for item in matching_items:\n",
    "    print(item.id)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8329547",
   "metadata": {},
   "source": [
    "### Copernicus dataspace Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3e58c3f2-1fbb-4e41-8f67-768d50225799",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'[]'\n"
     ]
    }
   ],
   "source": [
    "from oauthlib.oauth2 import BackendApplicationClient\n",
    "from requests_oauthlib import OAuth2Session\n",
    "\n",
    "# Your client credentials\n",
    "client_id = '###'\n",
    "client_secret = '###'\n",
    "\n",
    "# Create a session\n",
    "client = BackendApplicationClient(client_id=client_id)\n",
    "oauth = OAuth2Session(client=client)\n",
    "\n",
    "# Get token for the session\n",
    "token = oauth.fetch_token(token_url='https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token',\n",
    "                          client_secret=client_secret, include_client_id=True)\n",
    "\n",
    "def sentinelhub_compliance_hook(response):\n",
    "    response.raise_for_status()\n",
    "    return response\n",
    "\n",
    "oauth.register_compliance_hook(\"access_token_response\", sentinelhub_compliance_hook)\n",
    "# All requests using this session will have an access token automatically added\n",
    "resp = oauth.get(\"https://sh.dataspace.copernicus.eu/configuration/v1/wms/instances\")\n",
    "print(resp.content)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e066f967-b9d6-455f-ba84-da4103a0c5a0",
   "metadata": {},
   "source": [
    "## keycloak token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4028ccba-5bbb-4233-9c5f-738515e38aa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJYVUh3VWZKaHVDVWo0X3k4ZF8xM0hxWXBYMFdwdDd2anhob2FPLUxzREZFIn0.eyJleHAiOjE3MjgwNzAzNjksImlhdCI6MTcyODA2OTc2OSwianRpIjoiZTBkZmM4N2QtMmY0Ni00ZjNlLTk0ZTAtNTBiMDU0MzhmNDMxIiwiaXNzIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwiYXVkIjpbIkNMT1VERkVSUk9fUFVCTElDIiwiYWNjb3VudCJdLCJzdWIiOiIyYWMyNDIwZS1kM2MwLTQyNWYtYTY2OS1jYWY4YjA3N2QxZjMiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjZHNlLXB1YmxpYyIsInNlc3Npb25fc3RhdGUiOiJjZmI1MzY1NS05NDU2LTRjMjQtYmFjZi00N2I4NGU3MTVlZjQiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cHM6Ly9sb2NhbGhvc3Q6NDIwMCIsIioiLCJodHRwczovL3dvcmtzcGFjZS5zdGFnaW5nLWNkc2UtZGF0YS1leHBsb3Jlci5hcHBzLnN0YWdpbmcuaW50cmEuY2xvdWRmZXJyby5jb20iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iLCJkZWZhdWx0LXJvbGVzLWNkYXMiLCJjb3Blcm5pY3VzLWdlbmVyYWwiXX0sInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6IkFVRElFTkNFX1BVQkxJQyBvcGVuaWQgZW1haWwgcHJvZmlsZSBvbmRlbWFuZF9wcm9jZXNzaW5nIHVzZXItY29udGV4dCIsInNpZCI6ImNmYjUzNjU1LTk0NTYtNGMyNC1iYWNmLTQ3Yjg0ZTcxNWVmNCIsImdyb3VwX21lbWJlcnNoaXAiOlsiL2FjY2Vzc19ncm91cHMvdXNlcl90eXBvbG9neS9jb3Blcm5pY3VzX2dlbmVyYWwiLCIvb3JnYW5pemF0aW9ucy9kZWZhdWx0LTJhYzI0MjBlLWQzYzAtNDI1Zi1hNjY5LWNhZjhiMDc3ZDFmMy9yZWd1bGFyX3VzZXIiXSwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJSaXRodmlqIFBhc3VwdWxldGkiLCJvcmdhbml6YXRpb25zIjpbImRlZmF1bHQtMmFjMjQyMGUtZDNjMC00MjVmLWE2NjktY2FmOGIwNzdkMWYzIl0sInVzZXJfY29udGV4dF9pZCI6ImFhODlmYmFjLTRiNjYtNDM5ZS04ZGU0LWIzODJhNDYyYTJjZiIsImNvbnRleHRfcm9sZXMiOnt9LCJjb250ZXh0X2dyb3VwcyI6WyIvYWNjZXNzX2dyb3Vwcy91c2VyX3R5cG9sb2d5L2NvcGVybmljdXNfZ2VuZXJhbC8iLCIvb3JnYW5pemF0aW9ucy9kZWZhdWx0LTJhYzI0MjBlLWQzYzAtNDI1Zi1hNjY5LWNhZjhiMDc3ZDFmMy9yZWd1bGFyX3VzZXIvIl0sInByZWZlcnJlZF91c2VybmFtZSI6InJpdGh2aWoyMDE1QGdtYWlsLmNvbSIsImdpdmVuX25hbWUiOiJSaXRodmlqIiwiZmFtaWx5X25hbWUiOiJQYXN1cHVsZXRpIiwidXNlcl9jb250ZXh0IjoiZGVmYXVsdC0yYWMyNDIwZS1kM2MwLTQyNWYtYTY2OS1jYWY4YjA3N2QxZjMiLCJlbWFpbCI6InJpdGh2aWoyMDE1QGdtYWlsLmNvbSJ9.CNv-FDb8vnjZ6HaOM2eGqOtdfaxTjrI41w2L10qY06sauU_PRFkDBCryBs9BDVaVqKQzHmZ_21VQBboyJEb8EFNGkZsbPk7kopFLEXgincshKzM3cPV1uyfkGdM7ZwZpWxQiVvSxostyTSZObOfnZaspd_FFrBFw-_oYsEjYUxI_ek4aDaKU27SmH9rHXd-rAojINCGpB81aVPIHtZZSSdkMXn5x0jeLjiDMULM5kFzK0e_9EAFcpq8nk1OHw2A5TVsOu_SAbfOgecQIwoADLT0KV_L0tOlhz0VxDAnA8kKpJ7kX-XjINt5WZoc9_SWEDEvJosFy70kTIoreuherDA'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "def get_keycloak(username: str, password: str) -> str:\n",
    "    data = {\n",
    "        \"client_id\": \"cdse-public\",\n",
    "        \"username\": username,\n",
    "        \"password\": password,\n",
    "        \"grant_type\": \"password\",\n",
    "    }\n",
    "    try:\n",
    "        r = requests.post(\n",
    "            \"https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token\",\n",
    "            data=data,\n",
    "        )\n",
    "        r.raise_for_status()\n",
    "    except Exception as e:\n",
    "        raise Exception(\n",
    "            f\"Keycloak token creation failed. Reponse from the server was: {r.json()}\"\n",
    "        )\n",
    "    return r.json()[\"access_token\"]\n",
    "\n",
    "keycloak_token = get_keycloak('###','###')\n",
    "keycloak_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b71f60a0-3459-40bd-9609-96f1c918dad2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(81f48d1d-b98d-5f1b-9506-b24db5916ae5)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20180108T184332_20180108T184437_020067_022330_93EB.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(6ba51086-d986-5f4c-9567-4cfd99f26bd3)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20180306T101622_20180306T101722_020893_023D76_206D.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(c472fff8-1851-5508-a7e4-be2c28e3033c)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20180311T111524_20180311T111623_009983_01218C_F0B7.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(48650818-ebb9-5472-a006-5f70060cc281)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20180325T194759_20180325T194859_021176_024676_449C.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(2c915cd7-64f7-5544-9275-7332ddc14101)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20180707T122209_20180707T122309_022688_027552_A48E.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(0ddf8d13-bc0a-53b4-85bb-2be6030a8fba)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20180721T120156_20180721T120256_022892_027B98_56D9.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(99d8b5dd-58a1-5874-ac9b-b79d2540eceb)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20180829T113858_20180829T114003_012477_017029_869A.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(0a995fc9-aba9-5b47-aa7b-130aa33b0233)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20180902T124300_20180902T124400_012536_017204_4133.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(7bcc1342-8c18-5acf-bcc2-37e55712a1b8)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20180905T130632_20180905T130732_012580_017363_A7E8.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(994decf2-6a88-58a4-9e8c-b86c413dfd8f)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20180914T115342_20180914T115442_023694_02952F_CA97.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(df0fc0cc-cbd1-54df-b463-1f4529382b46)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20190607T082058_20190607T082158_027571_031C84_37CA.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(06aa948d-3064-5307-aa5c-e7173afd3839)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20190619T185115_20190619T185219_016769_01F8F8_AB64.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(fc5f7f7f-13d3-5cc9-bfe4-fe5cf8eeeb16)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20190808T120918_20190808T121018_017494_020E65_AA35.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(2ed11668-25e5-5ccf-80c7-50ae7c738884)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20190812T131548_20190812T131648_017553_021035_836B.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(388cdc4a-f6a1-5178-9e07-82e0d0e91ce4)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20190819T153633_20190819T153733_028640_033D98_D165.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(94ea1e1c-61c5-54fe-8898-e5c783faac10)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20190831T144552_20190831T144652_017831_0218EB_1D2C.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(322b444e-eb73-5943-9fcd-3dfce2a63264)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20190917T131550_20190917T131650_018078_022090_69A9.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(8035c2d6-c992-5ac2-ac08-fb3f79f02f35)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20200423T214037_20200423T214142_021277_02862A_4B23.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(461fe21e-1fd3-53a4-9717-edd56c8f5736)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20200619T122818_20200619T122923_033086_03D537_EA36.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(335cf75d-6a12-5a22-bc47-b55958423e22)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20200728T120112_20200728T120212_022671_02B07F_57F8.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(da1f145d-4e45-5582-8e48-554b54d77709)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20200819T104119_20200819T104219_022991_02BA50_1805.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(b5893fff-b491-5fae-9577-711c8c9581a8)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1A_EW_GRDM_1SDH_20200910T194821_20200910T194921_034301_03FCB0_E46C.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(63cb677a-412f-5232-89a6-7da95d4969e9)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20200922T172924_20200922T173024_023491_02C9FB_0F67.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(153dd8ad-11ea-502d-89aa-b77a3468ae33)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20201022T194736_20201022T194836_023930_02D7BE_BFD6.SAFE.zip\n",
      "Downloading from: https://zipper.dataspace.copernicus.eu/odata/v1/Products(4e19c1bb-0fde-56b5-8e05-1c207896194d)/$value\n",
      "Downloaded: C:/Users/r134p694_a/Cop_Matching/S1B_EW_GRDM_1SDH_20201104T123050_20201104T123132_024115_02DD70_0B82.SAFE.zip\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import requests\n",
    "\n",
    "# Ensure the directory exists\n",
    "output_dir = 'C:/Users/r134p694_a/Cop_Matching/'\n",
    "#os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# Function to download the item\n",
    "def download_item(item):\n",
    "    keycloak_token = get_keycloak('###','###')\n",
    "    # Access the 'PRODUCT' asset to get the download link\n",
    "    product_asset = item.assets.get('PRODUCT')\n",
    "    \n",
    "    if product_asset:\n",
    "        # Extract the product ID from the asset URL\n",
    "        product_id = product_asset.href.split('(')[1].split(')')[0]  # Extract the product ID\n",
    "        \n",
    "        # Construct the download URL for the ZIP file\n",
    "        download_url = f'https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value'\n",
    "\n",
    "        print(f\"Downloading from: {download_url}\")\n",
    "        \n",
    "        # Optional: Use headers for authorization if needed\n",
    "        headers = {\n",
    "            'Authorization': f\"Bearer {keycloak_token}\"  # Replace with your actual token if needed\n",
    "        }\n",
    "        \n",
    "        # Download the file\n",
    "        response = requests.get(download_url, headers=headers)\n",
    "        \n",
    "        if response.status_code == 200:\n",
    "            # Save the file as a ZIP\n",
    "            file_name = os.path.join(output_dir, f\"{item.id}.zip\")  # Change the extension to .zip\n",
    "            with open(file_name, 'wb') as f:\n",
    "                f.write(response.content)\n",
    "            print(f\"Downloaded: {file_name}\")\n",
    "        else:\n",
    "            print(f\"Failed to download {item.id} with status code: {response.status_code}\")\n",
    "            print(f\"Error Message: {response.text if response.text else 'No additional error message'}\")\n",
    "    else:\n",
    "        print(f\"No PRODUCT asset found for {item.id}\")\n",
    "\n",
    "#Iterate through filtered items and download each\n",
    "for item in matching_items:\n",
    "    download_item(item)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8057e80f-f821-43b7-a5f0-e1c2e345b48b",
   "metadata": {},
   "source": [
    "## Save as netcdfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "375e11ea-7065-48dd-a534-cd401a5f9a72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tiff files have been saved in NetCDF files!\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import zipfile\n",
    "import xarray as xr\n",
    "import rasterio\n",
    "\n",
    "# Paths\n",
    "source_dir = r'C:/Users/r134p694_a/Cop_Matching/'\n",
    "netcdf_dir = r'C:/Users/r134p694_a/Cop_Matching/netcdfs/'\n",
    "\n",
    "# Ensure the NetCDF directory exists\n",
    "os.makedirs(netcdf_dir, exist_ok=True)\n",
    "\n",
    "# Function to add TIFF data as a dataset in a NetCDF file\n",
    "def add_tiff_to_netcdf(tiff_path, ncfile, variable_name):\n",
    "    with rasterio.open(tiff_path) as src:\n",
    "        data = src.read(1)  # Read the first band\n",
    "        coords = {\n",
    "            'y': np.arange(data.shape[0]),\n",
    "            'x': np.arange(data.shape[1])\n",
    "        }\n",
    "        da = xr.DataArray(data, dims=['y', 'x'], coords=coords, name=variable_name)\n",
    "        da.to_netcdf(ncfile, mode='a')  # Append to existing NetCDF file\n",
    "\n",
    "# Loop through all ZIP files\n",
    "for file_name in os.listdir(source_dir):\n",
    "    if file_name.endswith('.zip'):\n",
    "        zip_path = os.path.join(source_dir, file_name)\n",
    "        base_name = os.path.splitext(file_name)[0]  # Name without extension\n",
    "        \n",
    "        # Create NetCDF file for each ZIP file\n",
    "        netcdf_path = os.path.join(netcdf_dir, f\"{base_name}.nc\")\n",
    "\n",
    "        with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n",
    "            hh_file = None\n",
    "            hv_file = None\n",
    "            for zip_info in zip_ref.infolist():\n",
    "                # Only process files in the \"measurement\" folder\n",
    "                if 'measurement' in zip_info.filename and zip_info.filename.endswith('.tiff'):\n",
    "                    # Check if the file name contains \"hh\" or \"hv\"\n",
    "                    if 'hh' in zip_info.filename.lower():\n",
    "                        hh_file = zip_info.filename\n",
    "                    elif 'hv' in zip_info.filename.lower():\n",
    "                        hv_file = zip_info.filename\n",
    "\n",
    "            # Extract and add the hh file to NetCDF\n",
    "            if hh_file:\n",
    "                hh_tiff_path = zip_ref.extract(hh_file, source_dir)\n",
    "                add_tiff_to_netcdf(hh_tiff_path, netcdf_path, 'hh')\n",
    "                os.remove(hh_tiff_path)  # Clean up extracted file\n",
    "\n",
    "            # Extract and add the hv file to NetCDF\n",
    "            if hv_file:\n",
    "                hv_tiff_path = zip_ref.extract(hv_file, source_dir)\n",
    "                add_tiff_to_netcdf(hv_tiff_path, netcdf_path, 'hv')\n",
    "                os.remove(hv_tiff_path)  # Clean up extracted file\n",
    "\n",
    "print(\"Tiff files have been saved in NetCDF files!\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "276d9b0a-0ec9-4fde-978d-603e87b35ce7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20180108T184332_20180108T184437_020067_022330_93EB.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20180306T101622_20180306T101722_020893_023D76_206D.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20180325T194759_20180325T194859_021176_024676_449C.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20180707T122209_20180707T122309_022688_027552_A48E.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20180721T120156_20180721T120256_022892_027B98_56D9.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20180914T115342_20180914T115442_023694_02952F_CA97.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20190607T082058_20190607T082158_027571_031C84_37CA.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20190819T153633_20190819T153733_028640_033D98_D165.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20200619T122818_20200619T122923_033086_03D537_EA36.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1A_EW_GRDM_1SDH_20200910T194821_20200910T194921_034301_03FCB0_E46C.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20180311T111524_20180311T111623_009983_01218C_F0B7.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20180829T113858_20180829T114003_012477_017029_869A.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20180902T124300_20180902T124400_012536_017204_4133.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20180905T130632_20180905T130732_012580_017363_A7E8.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20190619T185115_20190619T185219_016769_01F8F8_AB64.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20190808T120918_20190808T121018_017494_020E65_AA35.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20190812T131548_20190812T131648_017553_021035_836B.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20190831T144552_20190831T144652_017831_0218EB_1D2C.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20190917T131550_20190917T131650_018078_022090_69A9.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20200423T214037_20200423T214142_021277_02862A_4B23.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20200728T120112_20200728T120212_022671_02B07F_57F8.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20200819T104119_20200819T104219_022991_02BA50_1805.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20200922T172924_20200922T173024_023491_02C9FB_0F67.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20201022T194736_20201022T194836_023930_02D7BE_BFD6.SAFE.nc\n",
      "Downsampled file saved as: C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/S1B_EW_GRDM_1SDH_20201104T123050_20201104T123132_024115_02DD70_0B82.SAFE.nc\n",
      "Downsampling completed and saved in the downsampled directory.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import xarray as xr\n",
    "import numpy as np\n",
    "\n",
    "# Path to original and downsampled NetCDF files\n",
    "netcdf_dir = r'C:/Users/r134p694_a/Cop_Matching/netcdfs/'\n",
    "downsampled_dir = r'C:/Users/r134p694_a/Cop_Matching/netcdfs/downsampled/'\n",
    "\n",
    "# Ensure the downsampled directory exists\n",
    "os.makedirs(downsampled_dir, exist_ok=True)\n",
    "\n",
    "# Downsample function\n",
    "def downsample_image(image, scale_factor=2):\n",
    "    \"\"\"Downsamples the image using a 2x2 averaging kernel.\"\"\"\n",
    "    if image.shape[0] % scale_factor != 0 or image.shape[1] % scale_factor != 0:\n",
    "        image = image[:image.shape[0] - image.shape[0] % scale_factor,\n",
    "                      :image.shape[1] - image.shape[1] % scale_factor]\n",
    "\n",
    "    downsampled_image = image.reshape(\n",
    "        (image.shape[0] // scale_factor, scale_factor, \n",
    "         image.shape[1] // scale_factor, scale_factor)).mean(axis=(1, 3))\n",
    "    \n",
    "    return downsampled_image\n",
    "\n",
    "# Loop through all NetCDF files\n",
    "for file_name in os.listdir(netcdf_dir):\n",
    "    if file_name.endswith('.nc'):\n",
    "        netcdf_path = os.path.join(netcdf_dir, file_name)\n",
    "        downsampled_netcdf_path = os.path.join(downsampled_dir, file_name)  # Save to downsampled folder\n",
    "        \n",
    "        try:\n",
    "            # Open the original NetCDF file in read-only mode\n",
    "            with xr.open_dataset(netcdf_path, mode='r') as ds:\n",
    "                # Create a new dataset for downsampled data\n",
    "                downsampled_ds = xr.Dataset()\n",
    "\n",
    "                # Downsample and save the 'hh' and 'hv' datasets\n",
    "                if 'hh' in ds.variables:\n",
    "                    hh_image = ds['hh'].values  # Extract original image data\n",
    "                    downsampled_hh = downsample_image(hh_image)  # Downsample it\n",
    "                    # Create a new variable in the downsampled dataset with downsampled data\n",
    "                    downsampled_ds['hh'] = (('y', 'x'), downsampled_hh)\n",
    "\n",
    "                if 'hv' in ds.variables:\n",
    "                    hv_image = ds['hv'].values  # Extract original image data\n",
    "                    downsampled_hv = downsample_image(hv_image)  # Downsample it\n",
    "                    # Create a new variable in the downsampled dataset with downsampled data\n",
    "                    downsampled_ds['hv'] = (('y', 'x'), downsampled_hv)\n",
    "\n",
    "                # Carry over the attributes from the original dataset (optional)\n",
    "                downsampled_ds.attrs = ds.attrs\n",
    "\n",
    "                # Optionally, copy over other metadata or dimensions if needed\n",
    "                downsampled_ds['x'] = (('x',), np.arange(downsampled_hh.shape[1]))\n",
    "                downsampled_ds['y'] = (('y',), np.arange(downsampled_hh.shape[0]))\n",
    "\n",
    "                # Save the downsampled dataset to the new NetCDF file\n",
    "                downsampled_ds.to_netcdf(downsampled_netcdf_path)\n",
    "                print(f\"Downsampled file saved as: {downsampled_netcdf_path}\")\n",
    "                \n",
    "        except Exception as e:\n",
    "            print(f\"Error processing file {file_name}: {e}\")\n",
    "\n",
    "print(\"Downsampling completed and saved in the downsampled directory.\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}