fy3_l0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import h5py\n",
    "import struct\n",
    "import numpy as np\n",
    "import configparser\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['MAIN_DATA_PATH', 'FY3', 'HIMAWARI']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = configparser.ConfigParser()\n",
    "config.read('file_paths.ini')\n",
    "config.sections()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# File paths for FengYun-3E data.\n",
    "FY3E_DATA_FOLDERS_LOCATION = config['FY3']['FY3E_MAIN_DATA_LOCATION']\n",
    "FY3E_RAW_DATA_LOCATION = config['FY3']['FY3E_RAW_DATA_LOCATION']\n",
    "FY3E_L0_DATA_LOCATION = config['FY3']['FY3E_L0_DATA_LOCATION']\n",
    "FY3E_L1_DATA_LOCATION = config['FY3']['FY3E_L1_DATA_LOCATION']\n",
    "FY3E_L1_GOE_DATA_LOCATION = config['FY3']['FY3E_L1_GOE_DATA_LOCATION']\n",
    "FY3E_L1_IMAGE_DATA_LOCATION = config['FY3']['FY3E_L1_IMAGE_DATA_LOCATION']\n",
    "FY3E_L1_GEOREF_IMAGE_DATA_LOCATION = config['FY3']['FY3E_L1_GEOREF_IMAGE_DATA_LOCATION']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L0 files - ['E:/Satellite data/FY-3E/L0/Y3E_08534_230226070855_7860R_MERSI.DAT']\n",
      "\n",
      "GEO files - ['E:/Satellite data/FY-3E/L1/geo/FY3E_MERSI_GRAN_L1_20230228_2215_GEOQK_V0.HDF']\n",
      "\n",
      "Image files - ['E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230228_2215_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240301_0915_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230128_0435_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230128_0935_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230131_0520_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230131_0840_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230131_0845_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230131_1020_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230131_1745_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230131_2100_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230131_2250_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230326_1715_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230327_2200_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230331_2045_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230419_1800_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230430_1935_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20230728_0415_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231020_0620_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231028_0520_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231114_0955_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231116_1825_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231201_0440_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231204_0840_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231207_0920_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20231225_0705_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240104_0400_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240105_0655_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240129_2140_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240130_0405_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240220_0720_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240222_1905_0250M_V0.HDF', 'E:/Satellite data/FY-3E/L1/image/FY3E_MERSI_GRAN_L1_20240229_2155_0250M_V0.HDF']\n"
     ]
    }
   ],
   "source": [
    "fy3e_L0_files_paths = [FY3E_L0_DATA_LOCATION + file_name for file_name in os.listdir(FY3E_L0_DATA_LOCATION)]\n",
    "fy3e_l1_geo_files_paths = [FY3E_L1_GOE_DATA_LOCATION + file_name for file_name in os.listdir(FY3E_L1_GOE_DATA_LOCATION)]\n",
    "fy3e_l1_image_files_paths = [FY3E_L1_IMAGE_DATA_LOCATION + file_name for file_name in os.listdir(FY3E_L1_IMAGE_DATA_LOCATION)]\n",
    "\n",
    "print('L0 files -', fy3e_L0_files_paths)\n",
    "print()\n",
    "print('GEO files -', fy3e_l1_geo_files_paths)\n",
    "print()\n",
    "print('Image files -', fy3e_l1_image_files_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_uint12(data_chunk: bytes):\n",
    "    data = np.frombuffer(data_chunk, dtype=np.uint8)\n",
    "    fst_uint8, mid_uint8, lst_uint8 = np.reshape(data, (data.shape[0] // 3, 3)).astype(np.uint16).T\n",
    "    fst_uint12 = (fst_uint8 << 4) + (mid_uint8 >> 4)\n",
    "    snd_uint12 = ((mid_uint8 % 16) << 8) + lst_uint8\n",
    "    return np.reshape(np.concatenate((fst_uint12[:, None], snd_uint12[:, None]), axis=1), 2 * fst_uint12.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Optional\n",
    "from io import RawIOBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FY3EL0Header:\n",
    "    def __init__(self) -> None:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FY3EL0TransportPacket:\n",
    "    def __init__(self) -> None:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FY3EL0Data:\n",
    "    def __init__(self, \n",
    "                 header: FY3EL0Header, \n",
    "                 extra_packets: list[FY3EL0TransportPacket], \n",
    "                 packets: list[FY3EL0TransportPacket]) -> None:\n",
    "        \n",
    "        self.__header = header\n",
    "        self.__extra_packets = extra_packets\n",
    "        self.__packets = packets\n",
    "\n",
    "    @property\n",
    "    def header(self) -> FY3EL0Header:\n",
    "        return self.__header\n",
    "    \n",
    "    @property\n",
    "    def extra_packets(self) -> list[FY3EL0TransportPacket]:\n",
    "        return self.__extra_packets\n",
    "    \n",
    "    @property\n",
    "    def packets(self) -> list[FY3EL0TransportPacket]:\n",
    "        return self.__packets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FY3EL0Parser:\n",
    "    HEADER_SIZE = 1331906\n",
    "    SYNCHROSERIES_SIZE = 10\n",
    "    TRANSPORT_PACKET_SIZE = 9226\n",
    "\n",
    "    SYNCHROSERIES = b''\n",
    "    \n",
    "    def __init__(self, stream: RawIOBase) -> None:\n",
    "        self.stream = stream\n",
    "        \n",
    "    def parse_header(self) -> FY3EL0Header:\n",
    "        self.stream.read(FY3EL0Parser.HEADER_SIZE)\n",
    "\n",
    "        return FY3EL0Header()\n",
    "    \n",
    "    def parse_packet(self) -> Optional[FY3EL0TransportPacket]:\n",
    "        transport_packet = self.stream.read(FY3EL0Parser.TRANSPORT_PACKET_SIZE)\n",
    "\n",
    "        if transport_packet[:FY3EL0Parser.SYNCHROSERIES_SIZE] = \n",
    "\n",
    "    def parse(self) -> FY3EL0Data:\n",
    "        header = self.parse_header()\n",
    "        \n",
    "        extra_packets = []\n",
    "        while packet := self.parse_packet():\n",
    "            extra_packets.append(packet)\n",
    "\n",
    "        packets = []\n",
    "        while packet := self.parse_packet():\n",
    "            packets.append(packet)\n",
    "        \n",
    "        return FY3EL0Data(header, extra_packets, packets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = open(fy3e_L0_files_paths[0], 'br', buffering=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'x\\xb1\\x11\\x00FY3E  '\n"
     ]
    }
   ],
   "source": [
    "tmp = FY3EL0Parser(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}