Merge pull request #94 from lukasValentin/dev

Fix sign in offset for scaling and selection of reference bands for Landsat

.coveragerc

@@ -1,6 +1,6 @@
 [run]
 branch = True
-omit = */tests/*, eodal/downloader/, eodal/metadata/database/
+omit = */tests/*, */downloader/*, */metadata/*/database/*
 
 [report]
 exclude_lines =

CHANGELOG.rst

@@ -10,10 +10,19 @@ The format is based on `Keep a Changelog`_, and this project adheres to `Semanti
 
 Categories for changes are: Added, Changed, Deprecated, Removed, Fixed, Security.
 
+Version `0.2.3 < https://github.com/EOA-team/eodal/releases/tag/v0.2.4>`__
+--------------------------------------------------------------------------------
+
+Release date: 2023-12-01
+
+- Fixed: selection of reference bands fixed for Landsat (different sensors have different quality bands). Now, only those quality bands available for all sensor generations are retained in case there are multiple different Landsat sensor generation involved.
+- Fixed: sign of offset changed from - to + to be consistent with QGIS when scaling data.
+
+
 Version `0.2.3 < https://github.com/EOA-team/eodal/releases/tag/v0.2.3>`__
 --------------------------------------------------------------------------------
 
-Release date: 2023-10-XX
+Release date: 2023-11-24
 
 - Fixed: a fill value is now explicitly set to each `np.ma.masked_array`. If not (previous behavior) numpy would interfere a fill value that would often not be the no-data type of the raster. In the forthcoming release of EOdal, the fill-value will now equal the nodata value.
 - Fixed: the inference of the highest data type in a `RasterCollection` has been improved and extended to all `INT`,  `FLOAT`, and `COMPLEX` data types currently supported by numpy. In the previous version, all data types were cast to `numpy.float32` or even `numpy.float64` leading to an unnecessary high consumption of memory.

eodal/__meta__.py

@@ -15,4 +15,4 @@
 description = "Earth Observation Data Analysis Library"  # One-liner
 url = "https://github.com/EOA-team/eodal"  # your project home-page
 license = "GNU General Public License version 3"  # See https://choosealicense.com
-version = "0.2.3"
+version = "0.2.4"

eodal/core/band.py

@@ -2230,24 +2230,24 @@ def scale_data(
         scale, offset = self.scale, self.offset
         if self.is_masked_array:
             if pixel_values_to_ignore is None:
-                scaled_array = scale * self.values.data - offset
+                scaled_array = scale * self.values.data + offset
             else:
                 scaled_array = self.values.data.copy().astype(float)
                 scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] = scale * \
                     scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] \
-                    - offset
+                    + offset
             # reuse fill value
             fill_value = self.values.fill_value
             scaled_array = np.ma.MaskedArray(
                 data=scaled_array, mask=self.values.mask, fill_value=fill_value)
         elif self.is_ndarray:
             if pixel_values_to_ignore is None:
-                scaled_array = scale * self.values - offset
+                scaled_array = scale * self.values + offset
             else:
                 scaled_array = self.values.copy().astype(float)
                 scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] = scale * \
                     scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] \
-                    - offset
+                    + offset
         elif self.is_zarr:
             raise NotImplementedError()
 

eodal/core/sensors/sentinel2.py

@@ -117,7 +117,7 @@ def _get_gain_and_offset(in_dir: Union[str, Path]) -> Tuple[Number, Number]:
         # actual reflectance factor values.
         s2_offset = 0
         if baseline >= 400:
-            s2_offset = 0.1
+            s2_offset = -0.1
         return (s2_gain_factor, s2_offset)
 
     @staticmethod

eodal/mapper/mapper.py

@@ -737,9 +737,20 @@ def _load_scenes_collection(
         for _, scene in scoll:
             # loop over the bands and reproject them
             # onto the target grid
+            bands_to_remove = []
             for band_name, band in scene:
-                # get the reference band
-                reference_band = reference_scene[band_name]
+                # get the reference band. For some platforms, there
+                # might be different bands in the scene (different Landsat
+                # generations of sensors). In this case we have to continue.
+                try:
+                    reference_band = reference_scene[band_name]
+                except KeyError:
+                    # if band is not in reference scene, we memorize
+                    # the band name and continue. We delete the band
+                    # later to make sure the scenes in the collection
+                    # all have the same number of bands
+                    bands_to_remove.append(band_name)
+                    continue
                 geo_info = reference_band.geo_info
                 # get the transform of the destination extent
                 minx, maxy = total_bounds[0], total_bounds[-1]
@@ -769,6 +780,12 @@ def _load_scenes_collection(
                     dst_transform=dst_transform,
                     destination=destination,
                     dst_nodata=dst_nodata)
+            # remove bands if any
+            if len(bands_to_remove) > 0:
+                for band_to_remove in bands_to_remove:
+                    del scene[band_to_remove]
+                    if band_to_remove in scene.band_aliases:
+                        scene.band_aliases.remove(band_to_remove)
 
         self.data = scoll
 

setup.py

@@ -140,7 +140,7 @@ def run(self):
     name='eodal',
     # setup_requires=['setuptools_scm'],
     # use_scm_version={'version_scheme': 'python-simplified-semver'},
-    version='0.2.3',
+    version='0.2.4',
     description='The Earth Observation Data Analysis Library EOdal',
     long_description=long_description,
     long_description_content_type='text/markdown',

tests/core/test_raster_collection.py

@@ -136,7 +136,7 @@ def test_scale():
     ones = np.ones((100,100), dtype='uint16')
     band_name_ones = 'ones'
     scale = 100
-    offset = 2
+    offset = -2
     handler.add_band(
         Band,
         band_name=band_name_ones,
@@ -150,7 +150,7 @@ def test_scale():
     before_scaling = handler.get_values()
     handler.scale(inplace=True)
     after_scaling = handler.get_values()
-    assert (after_scaling == scale * before_scaling - offset).all(), 'values not scaled correctly'
+    assert (after_scaling == scale * before_scaling + offset).all(), 'values not scaled correctly'
     assert (after_scaling == 98).all(), 'wrong value after scaling'