Mag l1a compression

.pre-commit-config.yaml

@@ -38,7 +38,7 @@ repos:
       - id: numpydoc-validation
         exclude: '^imap_processing/tests/|.*test.*'
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.10.0'
+    rev: 'v1.10.1'
     hooks:
       - id: mypy
         exclude: .*(tests|docs).*

examples/Dockerfile.processing

@@ -4,7 +4,8 @@ FROM public.ecr.aws/docker/library/python:3.10-slim
 # TODO: delete this section once imap_processing is released
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y git
-RUN pip install git+https://github.com/IMAP-Science-Operations-Center/imap_processing.git@dev
+#RUN pip install git+https://github.com/IMAP-Science-Operations-Center/imap_processing.git@dev
+RUN pip install git+https://github.com/maxinelasp/imap_processing.git@mag_l1a_compression
 
 # Uncomment this once imap_processing is released
 # RUN pip install imap_processing

imap_processing/mag/constants.py

@@ -2,6 +2,8 @@
 
 from enum import Enum
 
+import numpy as np
+
 
 class DataMode(Enum):
     """
@@ -55,3 +57,51 @@ class PrimarySensor(Enum):
 
     MAGO = 0
     MAGI = 1
+
+
+FIBONACCI_SEQUENCE = [
+    1,
+    2,
+    3,
+    5,
+    8,
+    13,
+    21,
+    34,
+    55,
+    89,
+    144,
+    233,
+    377,
+    610,
+    987,
+    1597,
+    2584,
+    4181,
+    6765,
+    10946,
+    17711,
+    28657,
+    46368,
+    75025,
+    121393,
+    196418,
+    317811,
+    514229,
+    832040,
+    1346269,
+    2178309,
+    3524578,
+    5702887,
+    9227465,
+    14930352,
+    24157817,
+    39088169,
+    63245986,
+    102334155,
+    165580141,
+]
+
+MAX_FINE_TIME = np.iinfo(np.uint16).max  # maximum 16 bit unsigned int
+AXIS_COUNT = 3
+RANGE_BIT_WIDTH = 2

imap_processing/mag/l0/mag_l0_data.py

@@ -106,19 +106,13 @@ def __post_init__(self) -> None:
         Also convert encoded "VECSEC" (vectors per second) into proper vectors per
         second values.
         """
-        # Convert string output from space_packet_parser to numpy array of
-        # big-endian bytes
-        self.VECTORS = np.frombuffer(
-            int(self.VECTORS, 2).to_bytes(len(self.VECTORS) // 8, "big"),  # type: ignore[arg-type]
-            # TODO Check MYPY Error: Argument 1 to "int" has incompatible type
-            # "Union[ndarray[Any, Any], str]"; expected "Union[str, bytes, bytearray]"
-            dtype=np.dtype(">b"),
-        )
-
-        # Remove buffer from end of vectors. Vector data needs to be in 50 bit chunks,
-        # and may have an extra byte at the end from CCSDS padding.
-        if len(self.VECTORS) % 2:
-            self.VECTORS = self.VECTORS[:-1]
+        if isinstance(self.VECTORS, str):
+            # Convert string output from space_packet_parser to numpy array of
+            # big-endian bytes
+            self.VECTORS = np.frombuffer(
+                int(self.VECTORS, 2).to_bytes(len(self.VECTORS) // 8, "big"),
+                dtype=np.dtype(">B"),
+            )
 
         self.PRI_VECSEC = 2**self.PRI_VECSEC
         self.SEC_VECSEC = 2**self.SEC_VECSEC

imap_processing/mag/l1a/mag_l1a.py

@@ -143,9 +143,6 @@ def process_packets(
     mago = {}
 
     for mag_l0 in mag_l0_list:
-        if mag_l0.COMPRESSION:
-            raise NotImplementedError("Unable to process compressed data")
-
         primary_start_time = TimeTuple(mag_l0.PRI_COARSETM, mag_l0.PRI_FNTM)
         secondary_start_time = TimeTuple(mag_l0.SEC_COARSETM, mag_l0.SEC_FNTM)
 
@@ -181,14 +178,11 @@ def process_packets(
         # now we know the number of secs of data in the packet, and the data rates of
         # each sensor, we can calculate how much data is in this packet and where the
         # byte boundaries are.
-
         primary_vectors, secondary_vectors = MagL1a.process_vector_data(
-            mag_l0.VECTORS.astype(dtype=np.int32),  # type: ignore[union-attr]
-            # TODO Maybe Change, Item "str" of "Union[Any, str]"
-            #  has no attribute "astype"
-            # this is because mypy expects both to have the attributes
+            mag_l0.VECTORS,  # type: ignore
             primary_packet_data.total_vectors,
             secondary_packet_data.total_vectors,
+            mag_l0.COMPRESSION,
         )
 
         primary_timestamped_vectors = MagL1a.calculate_vector_time(