Merge branch 'dev' into codice-l1-updates

.pre-commit-config.yaml

@@ -41,9 +41,5 @@ repos:
     rev: 'v1.10.0'
     hooks:
       - id: mypy
-        pass_filenames: false
-        args: [ ., --strict, --explicit-package-bases,
-                  --disable-error-code, import-untyped,
-                  --disable-error-code, import-not-found,
-                  --disable-error-code, no-untyped-call,
-                  --disable-error-code, type-arg ]
+        exclude: .*(tests|docs).*
+        additional_dependencies: [ numpy==1.26.4 ]

docs/source/conf.py

@@ -113,6 +113,9 @@
     (r"py:.*", r".*InitVar*"),
     (r"py:.*", r".*.glows.utils.constants.TimeTuple.*"),
     (r"py:.*", r".*glows.utils.constants.DirectEvent.*"),
+    (r"py:.*", r".*numpy.int.*"),
+    (r"py:.*", r".*np.ndarray.*"),
+    (r"py:.*", r".*numpy._typing._array_like._ScalarType_co.*"),
 ]
 
 # Ignore the inherited members from the <instrument>APID IntEnum class

docs/source/data-access-api/index.rst

@@ -11,14 +11,21 @@ The SDC provides a REST API that allows users to upload and download files, as
 well as query for file metadata. The following documentation describes the
 various endpoints that are supported and how to use them.
 
-The API can be accessed from the following URL: https://api.dev.imap-mission.com
-
+*Note: Several sections and links begin with* [WIP]. *As development on the API is ongoing, this indicates
+that the full implementation of the functionality is yet to be completed.*
 
+The API can be accessed from the following URL [WIP]: https://api.dev.imap-mission.com
 
 .. openapi:: openapi.yml
    :group:
    :include: /upload
 
+When uploading files to the API, ensure these files are stored properly in a ``data`` directory. Then,
+ensure your working directory is one level above the ``data`` directory in order to properly upload files.
+
+[WIP] Certain ancillary files can also be uploaded to the API. For more specific information regarding these files, visit
+`Ancillary Files <https://imap-processing.readthedocs.io/en/latest/data-access-api/calibration-files.html>`_
+
 **Example Usage:**
 
 .. code-block:: bash
@@ -45,6 +52,13 @@ The API can be accessed from the following URL: https://api.dev.imap-mission.com
    :group:
    :include: /download
 
+It is important to note that your working directory will be established as the default directory. I.e, the ``data``
+directory--which files are downloaded to--will automatically be placed in this file path. Choose your working directory
+accordingly to suit your desires.
+
+When downloading a file from the API, different folders within the ``data`` directory will be made to better
+organize the files. See the example file path: ``data/imap/swe/l0/2024/01/imap_swe_l0_sci_20240105_20240105_v00-01.pkts``
+
 **Example Usage:**
 
 .. code-block:: bash

docs/source/data-access-api/openapi.yml

@@ -52,6 +52,7 @@ paths:
                 items:
                   type: string
 
+
   '/download/{filepath}':
     get:
       tags:
@@ -143,7 +144,8 @@ paths:
             type: string
         - in: query
           name: version
-          description: The version of data product in the format ``vXX-YY`` (e.g. ``v01-01``).
+          description: The version of data product in the format ``vNNN`` (e.g. ``v001``). You can also choose to
+            query ``--version latest`` in order to receive the most recent version of a file.
           required: false
           schema:
             type: string

imap_processing/ccsds/excel_to_xtce.py

@@ -226,7 +226,9 @@ def _create_container_sets(self) -> None:
                 if pd.isna(row.get("packetName")):
                     # This is a poorly formatted row, skip it
                     continue
-                name = f"{row['packetName']}_{row['mnemonic']}"
+                # separate the packet name and mnemonic with a period
+                # a hyphen is sometimes in the packet name or mnemonic already
+                name = f"{row['packetName']}.{row['mnemonic']}"
                 parameter_ref_entry = Et.SubElement(
                     packet_entry_list, "xtce:ParameterRefEntry"
                 )
@@ -247,7 +249,7 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None:
         """
         parameter = Et.SubElement(self._parameter_set, "xtce:Parameter")
         # Combine the packet name and mnemonic to create a unique parameter name
-        name = f"{row['packetName']}_{row['mnemonic']}"
+        name = f"{row['packetName']}.{row['mnemonic']}"
         parameter.attrib["name"] = name
         # UINT8, ...
         parameter.attrib["parameterTypeRef"] = name
@@ -262,7 +264,7 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None:
         length_in_bits = int(row["lengthInBits"])
 
         # Add the parameterTypeRef for this row
-        if "UINT" in row["dataType"]:
+        if "UINT" in row["dataType"] or "FILL" in row["dataType"]:
             parameter_type = Et.SubElement(
                 self._parameter_type_set, "xtce:IntegerParameterType"
             )
@@ -283,6 +285,15 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None:
             encoding.attrib["sizeInBits"] = str(length_in_bits)
             encoding.attrib["encoding"] = "signed"
 
+        elif "FLOAT" in row["dataType"]:
+            parameter_type = Et.SubElement(
+                self._parameter_type_set, "xtce:FloatParameterType"
+            )
+            parameter_type.attrib["name"] = name
+            encoding = Et.SubElement(parameter_type, "xtce:FloatDataEncoding")
+            encoding.attrib["sizeInBits"] = str(length_in_bits)
+            encoding.attrib["encoding"] = "IEEE-754"
+
         elif "BYTE" in row["dataType"]:
             parameter_type = Et.SubElement(
                 self._parameter_type_set, "xtce:BinaryParameterType"
@@ -311,6 +322,8 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None:
             # TODO: Do we want to allow fixed length values?
             # fixed_value = Et.SubElement(size_in_bits, "xtce:FixedValue")
             # fixed_value.text = str(row["lengthInBits"])
+        else:
+            raise ValueError(f"Unknown data type for {name}: {row['dataType']}")
 
         if row["convertAs"] == "ANALOG":
             # Go look up the conversion in the AnalogConversions tab

imap_processing/cdf/__init__.py

@@ -1,5 +0,0 @@
-from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-
-# Load Epoch CDF attributes
-cdf_manager = ImapCdfAttributes()
-epoch_attrs = cdf_manager.get_variable_attributes("epoch")

imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml

@@ -36,23 +36,6 @@ energy_attrs:
     VALIDMAX: 127
     VAR_TYPE: support_data
 
-epoch_attrs:
-    <<: *default
-    CATDESC: Time, number of nanoseconds since J2000 with leap seconds included
-    FIELDNAM: epoch
-    LABLAXIS: epoch
-    FILLVAL: -9223372036854775808
-    FORMAT: " " # Supposedly not required, fails in xarray_to_cdf
-    VALIDMIN: -9223372036854775808
-    VALIDMAX: 9223372036854775807
-    UNITS: ns
-    VAR_TYPE: support_data
-    SCALETYP: linear
-    MONOTON: INCREASE
-    TIME_BASE: J2000
-    TIME_SCALE: Terrestrial Time
-    REFERENCE_POSITION: Rotating Earth Geoid
-
 # <=== Labels ===>
 energy_label:
     CATDESC: Energy per charge (E/q) sweeping step

imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml

@@ -32,23 +32,6 @@ energy_attrs:
     VALIDMAX: 127
     VAR_TYPE: support_data
 
-epoch_attrs:
-    <<: *default
-    CATDESC: Time, number of nanoseconds since J2000 with leap seconds included
-    FIELDNAM: epoch
-    LABLAXIS: epoch
-    FILLVAL: -9223372036854775808
-    FORMAT: " " # Supposedly not required, fails in xarray_to_cdf
-    VALIDMIN: -9223372036854775808
-    VALIDMAX: 9223372036854775807
-    UNITS: ns
-    VAR_TYPE: support_data
-    SCALETYP: linear
-    MONOTON: INCREASE
-    TIME_BASE: J2000
-    TIME_SCALE: Terrestrial Time
-    REFERENCE_POSITION: Rotating Earth Geoid
-
 # <=== Labels ===>
 energy_label:
     CATDESC: Energy per charge (E/q) sweeping step

imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml

@@ -1,8 +1,17 @@
-Project: STP>Solar-Terrestrial Physics
+# Reference for sections: https://spdf.gsfc.nasa.gov/istp_guide/gattributes.html
+# Reference from IBEX: https://spdf.gsfc.nasa.gov/pub/software/cdawlib/0SKELTABLES/ibex_h3_ena_lo_r08_omni_f3-gdf-maps_5yr_00000000_v01.skt
+Project: STP>Solar Terrestrial Probes
 Source_name: IMAP>Interstellar Mapping and Acceleration Probe
 Discipline: Solar Physics>Heliospheric Physics
-# TODO: CDF docs say this value should be IMAP
-Mission_group: IMAP>Interstellar Mapping and Acceleration Probe
-PI_name: Dr. David J. McComas
-PI_affiliation: Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540
+Mission_group: IMAP
+PI_name: Prof. David J. McComas
+PI_affiliation: Princeton University
 File_naming_convention: source_descriptor_datatype_yyyyMMdd_vNNN
+Acknowledgement: >
+  Please acknowledge the IMAP Mission Principal Investigator, Prof. David J. McComas of Princeton University.
+Rules_of_use: >
+  All IMAP data products are publicly released and citable for use in publications.
+  Please consult the IMAP team publications and personnel for further details
+  on production, processing, and usage of these data.
+LINK_TITLE: IMAP The Interstellar Mapping and Acceleration Probe
+HTTP_LINK: https://imap.princeton.edu/

imap_processing/cdf/config/imap_glows_global_cdf_attrs.yaml

@@ -2,14 +2,15 @@ instrument_base: &instrument_base
   Descriptor: GLOWS>GLObal Solar Wind Structure
   TEXT: >
     The GLObal Solar Wind Structure (GLOWS) is a non-imaging single-pixel Lyman-alpha
-    photometer that will be used to observe the sky distribution of the helioglow to
-    better understand the evolution of the solar wind structure.
-    The Lyman-alpha photon counts from these observations can be used to build a more
-    comprehensive picture of the solar wind structure and how it changes through the
-    solar cycles.
+    photometer to investigate the global heliolatitudinal structure of the solar wind
+    and its evolution during the solar cycle. Additionally, GLOWS investigates the
+    distribution of interstellar neutral hydrogen (ISN H) and the solar radiation
+    pressure acting on ISN H. The objectives of GLOWS are accomplished by observation
+    of the modulation of heliospheric backscatter glow of ISN H (the helioglow)
+    along a scanning circle in the sky.
     GLOWS design and assembly is led by the Space Research Center, Warsaw, Poland
     (CBK PAN). See https://imap.princeton.edu/instruments/glows for more details.
-  Instrument_type: Particles (space)
+  Instrument_type: Imagers (space)
 
 imap_glows_l1a_hist:
   <<: *instrument_base