Merge pull request #388 from LCOGT/fix/header-only-fits

Fix to fpacking extensions when they have no data and only a header

CHANGES.md

@@ -1,3 +1,7 @@
+1.15.2 (2023-04-12)
+-------------------
+- Fix to fpacking data when the image data array is None
+
 1.15.1 (2024-02-29)
 -------------------
 - Minor fixes in photometry when there are bad pixels near the image edges

banzai/data.py

@@ -85,8 +85,8 @@ def to_fits(self, context) -> Union[fits.HDUList, list]:
 
 
 class HeaderOnly(Data):
-    def __init__(self, meta: Union[dict, fits.Header]):
-        super().__init__(data=np.zeros(0), meta=meta, memmap=False)
+    def __init__(self, meta: Union[dict, fits.Header], name):
+        super().__init__(data=np.zeros(0), meta=meta, memmap=False, name=name)
 
     def to_fits(self, context):
         return fits.HDUList([fits.ImageHDU(data=None, header=self.meta)])

banzai/lco.py

@@ -412,7 +412,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]:
                     for hdu in fits_hdu_list if hdu.data is not None):
             for hdu in fits_hdu_list:
                 if hdu.data is None or hdu.data.size == 0:
-                    hdu_list.append(HeaderOnly(meta=hdu.header))
+                    hdu_list.append(HeaderOnly(meta=hdu.header, name=hdu.header.get('EXTNAME')))
                 else:
                     hdu_list.append(self.data_class(data=hdu.data, meta=hdu.header, name=hdu.header.get('EXTNAME')))
         else:
@@ -424,7 +424,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]:
                     continue
                 # Otherwise parse the fits file into a frame object and the corresponding data objects
                 if hdu.data is None or hdu.data.size == 0:
-                    hdu_list.append(HeaderOnly(meta=hdu.header))
+                    hdu_list.append(HeaderOnly(meta=hdu.header, name=hdu.header.get('EXTNAME')))
                     primary_hdu = hdu
                 elif isinstance(hdu, fits.BinTableHDU):
                     hdu_list.append(DataTable(data=Table(hdu.data), meta=hdu.header, name=hdu.header.get('EXTNAME')))
@@ -601,7 +601,7 @@ def _munge_data_cube(hdu):
         :return: List CCDData objects
         """
         # The first extension gets to be a header only object
-        hdu_list = [HeaderOnly(meta=hdu.header)]
+        hdu_list = [HeaderOnly(meta=hdu.header, name=hdu.header.get('EXTNAME'))]
 
         # We need to properly set the datasec and detsec keywords in case we didn't read out the
         # middle row (the "Missing Row Problem").

banzai/tests/test_munge.py

@@ -23,7 +23,7 @@ def test_when_has_partial_coefficients():
     for i in range(2):
         for j in range(2):
             header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = 1.0
-    hdu_list = [HeaderOnly(meta=header)] + [FakeCCDData() for i in range(4)]
+    hdu_list = [HeaderOnly(meta=header, name='')] + [FakeCCDData() for i in range(4)]
     with pytest.raises(MissingCrosstalkCoefficients):
         LCOFrameFactory._init_crosstalk(FakeLCOObservationFrame(hdu_list=hdu_list))
 
@@ -34,7 +34,7 @@ def test_when_has_coefficients():
     for i in range(4):
         for j in range(4):
             header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = 1.0
-    hdu_list = [HeaderOnly(meta=header)] + [FakeCCDData() for i in range(4)]
+    hdu_list = [HeaderOnly(meta=header, name='')] + [FakeCCDData() for i in range(4)]
     LCOFrameFactory._init_crosstalk(FakeLCOObservationFrame(hdu_list=hdu_list))
 
 
@@ -44,7 +44,7 @@ def test_defaults_do_not_override_header():
         for j in range(4):
             header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = 1.0
 
-    hdu_list = [HeaderOnly(meta=header)] + [FakeCCDData() for i in range(4)]
+    hdu_list = [HeaderOnly(meta=header, name='')] + [FakeCCDData() for i in range(4)]
     fake_image = FakeLCOObservationFrame(hdu_list=hdu_list)
     LCOFrameFactory._init_crosstalk(fake_image)
 

banzai/tests/utils.py

@@ -102,8 +102,8 @@ def handles_inhomogeneous_set(stagetype, context, keyword, value, calibration_ma
     stage = stagetype(context)
     kwargs = {keyword: value}
     if calibration_maker:
-        images = [LCOCalibrationFrame(hdu_list=[HeaderOnly(meta=kwargs)])]
-        images += [LCOCalibrationFrame(hdu_list=[HeaderOnly()]) for x in range(6)]
+        images = [LCOCalibrationFrame(hdu_list=[HeaderOnly(meta=kwargs, name='')])]
+        images += [LCOCalibrationFrame(hdu_list=[HeaderOnly(meta={}, name=''),]) for x in range(6)]
         images = stage.do_stage(images)
         assert len(images) == 0
     else:

banzai/utils/fits_utils.py

@@ -199,12 +199,15 @@ def unpack(compressed_hdulist: fits.HDUList) -> fits.HDUList:
         starting_extension = 1
     for hdu in compressed_hdulist[starting_extension:]:
         if isinstance(hdu, fits.CompImageHDU):
-            data_type = str(hdu.data.dtype)
-            if 'int' == data_type[:3]:
-                data_type = getattr(np, 'u' + data_type)
-                data = np.array(hdu.data, data_type)
+            if hdu.data is None:
+                data = hdu.data
             else:
-                data = np.array(hdu.data, hdu.data.dtype)
+                data_type = str(hdu.data.dtype)
+                if 'int' == data_type[:3]:
+                    data_type = getattr(np, 'u' + data_type)
+                    data = np.array(hdu.data, data_type)
+                else:
+                    data = np.array(hdu.data, hdu.data.dtype)
             hdulist.append(fits.ImageHDU(data=data, header=hdu.header))
         elif isinstance(hdu, fits.BinTableHDU):
             hdulist.append(fits.BinTableHDU(data=hdu.data, header=hdu.header))
@@ -223,7 +226,11 @@ def pack(uncompressed_hdulist: fits.HDUList, lossless_extensions: Iterable) -> f
             quantize_level = 1e9
         else:
             quantize_level = 64
-        compressed_hdu = fits.CompImageHDU(data=np.ascontiguousarray(uncompressed_hdulist[0].data),
+        if uncompressed_hdulist[0].data is None:
+            data = None
+        else:
+            data = np.ascontiguousarray(uncompressed_hdulist[0].data)
+        compressed_hdu = fits.CompImageHDU(data=data,
                                            header=uncompressed_hdulist[0].header, quantize_level=quantize_level,
                                            quantize_method=1)
         hdulist = [primary_hdu, compressed_hdu]
@@ -234,7 +241,11 @@ def pack(uncompressed_hdulist: fits.HDUList, lossless_extensions: Iterable) -> f
                 quantize_level = 1e9
             else:
                 quantize_level = 64
-            compressed_hdu = fits.CompImageHDU(data=np.ascontiguousarray(hdu.data), header=hdu.header,
+            if hdu.data is None:
+                data = None
+            else:
+                data = np.ascontiguousarray(hdu.data)
+            compressed_hdu = fits.CompImageHDU(data=data, header=hdu.header,
                                                quantize_level=quantize_level, quantize_method=1)
             hdulist.append(compressed_hdu)
         else:

banzai/utils/realtime_utils.py

@@ -86,7 +86,7 @@ def need_to_process_image(file_info, context):
     if 'frameid' in file_info:
         try:
             factory = import_utils.import_attribute(context.FRAME_FACTORY)()
-            test_image = factory.observation_frame_class(hdu_list=[HeaderOnly(file_info)],
+            test_image = factory.observation_frame_class(hdu_list=[HeaderOnly(file_info, name='')],
                                                          file_path=file_info['filename'])
             test_image.instrument = factory.get_instrument_from_header(file_info, db_address=context.db_address)
             if image_utils.get_reduction_level(test_image.meta) != '00':