GITC-8325: Properly handle data values outside of colormap range

CHANGELOG.md

@@ -11,7 +11,13 @@ Changelog](http://keepachangelog.com/en/1.0.0/).
 * GitHub release notes for HyBIG will now include the commit history for that
   release.
 
-## [v2.4.2] - Unreleased
+## [v2.5.0] - Unreleased
+
+### Changed
+
+* Correctly handle clipping behavior for values outside the colormap range
+
+## [v2.4.2] - 2025-10-28
 
 ### Changed
 

docker/service_version.txt

@@ -1 +1 @@
-2.4.2
+2.5.0

hybig/browse.py

@@ -312,7 +312,7 @@ def scale_grey_1band(data_array: DataArray) -> tuple[ndarray, ColorMap]:
     normalized_data = norm(band) * 254.0
 
     # Set any missing to missing
-    normalized_data[np.isnan(normalized_data)] = NODATA_IDX
+    normalized_data[np.isnan(band)] = NODATA_IDX
 
     grey_colormap = greyscale_colormap()
     raster_data = np.expand_dims(np.round(normalized_data).data, 0)
@@ -342,7 +342,7 @@ def scale_grey_1band_to_rgb(data_array: DataArray) -> tuple[ndarray, None]:
     normalized_data = norm(band) * 254.0
 
     # Set any missing to 0 (black), no transparency
-    normalized_data[np.isnan(normalized_data)] = 0
+    normalized_data[np.isnan(band)] = 0
 
     grey_data = np.round(normalized_data).astype('uint8')
     rgb_data = np.stack([grey_data, grey_data, grey_data], axis=0)
@@ -367,8 +367,17 @@ def scale_paletted_1band_to_rgb(
     if palette.ndv is not None:
         nodata_color = palette.color_to_color_entry(palette.ndv, with_alpha=True)
 
+    # Store NaN mask before normalization
+    nan_mask = np.isnan(band)
+
+    # Replace NaN with first level to avoid issues during normalization
+    band_clean = np.where(nan_mask, levels[0], band)
+
     # Apply normalization to get palette indices
-    indexed_band = norm(band)
+    indexed_band = norm(band_clean)
+
+    # Clip indices to valid range [0, len(colors)-1]
+    indexed_band = np.clip(indexed_band, 0, len(colors) - 1)
 
     # Create RGB output array
     height, width = band.shape
@@ -381,8 +390,7 @@ def scale_paletted_1band_to_rgb(
         rgb_array[1, mask] = color[1]  # Green
         rgb_array[2, mask] = color[2]  # Blue
 
-    # Handle NaN/nodata values
-    nan_mask = np.isnan(band)
+    # Handle NaN/nodata values (overwrite any color assignment)
     if nan_mask.any():
         rgb_array[0, nan_mask] = nodata_color[0]
         rgb_array[1, nan_mask] = nodata_color[1]
@@ -399,6 +407,10 @@ def scale_paletted_1band(
     Use the palette's levels and values, transform the input data_array into
     the correct levels indexed from 0-255 return the scaled array along side of
     a colormap corresponding to the new levels.
+
+    Values below the minimum palette level are clipped to the lowest color.
+    Values above the maximum palette level are clipped to the highest color.
+    Only NaN values are mapped to the nodata index.
     """
     global DST_NODATA
     band = data_array[0, :, :]
@@ -413,31 +425,38 @@ def scale_paletted_1band(
     nodata_color = (0, 0, 0, 0)
     if palette.ndv is not None:
         nodata_color = palette.color_to_color_entry(palette.ndv, with_alpha=True)
-        color_list = list(palette.pal.values())
-        try:
-            DST_NODATA = color_list.index(palette.ndv)
-            # ndv is included in the list of colors, so no need to add nodata_color
-            # to the list
-        except ValueError:
-            # ndv is not an index in the color palette, therefore it should be
-            # index 0, which is the default for a ColorPalette when using
-            # palette.get_all_keys()
+        # Check if nodata color already exists in palette
+        if palette.ndv in palette.pal.values():
+            DST_NODATA = list(palette.pal.values()).index(palette.ndv)
+            # Don't add nodata_color; it's already in colors
+        else:
+            # Nodata not in palette, add it at the beginning
             DST_NODATA = 0
             colors = [nodata_color, *colors]
     else:
         # if there is no ndv, add one to the end of the colormap
+        DST_NODATA = len(colors)
         colors = [*colors, nodata_color]
-        DST_NODATA = len(colors) - 1
 
-    scaled_band = norm(band)
+    nan_mask = np.isnan(band)
+    band_clean = np.where(nan_mask, levels[0], band)
+    scaled_band = norm(band_clean)
+
     if DST_NODATA == 0:
         # boundary norm indexes [0, levels) by default, so if the NODATA index is 0,
         # all the palette indices need to be incremented by 1.
-        scaled_band += 1
+        scaled_band = scaled_band + 1
+
+    # Clip to valid palette range (excluding nodata index)
+    if DST_NODATA == 0:
+        # Palette occupies indices 1 to len(colors)-1
+        scaled_band = np.clip(scaled_band, 1, len(colors) - 1)
+    else:
+        # Palette occupies indices 0 to DST_NODATA-1
+        scaled_band = np.clip(scaled_band, 0, DST_NODATA - 1)
 
-    # Set underflow and nan values to nodata index
-    scaled_band[scaled_band == -1] = DST_NODATA
-    scaled_band[np.isnan(band)] = DST_NODATA
+    # Only set NaN values to nodata index
+    scaled_band[nan_mask] = DST_NODATA
 
     color_map = colormap_from_colors(colors)
     raster_data = np.expand_dims(scaled_band.data, 0)

tests/unit/test_browse.py

@@ -776,6 +776,197 @@ def test_palette_from_remote_colortable(self, mock_get):
                 ),
             )
 
+    def test_scale_paletted_1band_clips_underflow_values(self):
+        """Test that values below the palette min are clipped to lowest color."""
+        from hybig.browse import scale_paletted_1band
+
+        # Create test data with values below the palette minimum
+        # Palette covers 100-400, but data includes -50 and 50
+        data_with_underflow = np.array(
+            [
+                [-50, 50, 100, 200],
+                [100, 200, 300, 400],
+                [50, 100, 200, 300],
+                [-100, 0, 150, 250],
+            ]
+        ).astype('float64')
+        ds = DataArray(data_with_underflow).expand_dims('band')
+
+        # Expected: underflow values (-50, 50, 0, -100) should map to index 0
+        # which is the lowest color (red at value 100)
+        expected_raster = np.array(
+            [
+                [
+                    [0, 0, 0, 1],  # -50, 50 -> 0 (red), 100->0, 200->1
+                    [0, 1, 2, 3],  # 100->0, 200->1, 300->2, 400->3
+                    [0, 0, 1, 2],  # 50, 100 -> 0, 200->1, 300->2
+                    [0, 0, 0, 1],  # -100, 0 -> 0, 150->0, 250->1
+                ],
+            ],
+            dtype='uint8',
+        )
+
+        image_palette = convert_colormap_to_palette(self.colormap)
+        actual_raster, _ = scale_paletted_1band(ds, image_palette)
+        assert_array_equal(expected_raster, actual_raster, strict=True)
+
+    def test_scale_paletted_1band_clips_overflow_values(self):
+        """Test that values above the palette max are clipped to highest color."""
+        from hybig.browse import scale_paletted_1band
+
+        # Create test data with values above the palette maximum
+        # Palette covers 100-400, but data includes 500 and 1000
+        data_with_overflow = np.array(
+            [
+                [100, 200, 300, 400],
+                [400, 500, 600, 1000],
+                [200, 300, 400, 500],
+                [300, 350, 400, 800],
+            ]
+        ).astype('float64')
+        ds = DataArray(data_with_overflow).expand_dims('band')
+
+        # Expected: overflow values (500, 600, 1000, 800) should map to index 3
+        # which is the highest color (blue at value 400)
+        expected_raster = np.array(
+            [
+                [
+                    [0, 1, 2, 3],  # 100->0, 200->1, 300->2, 400->3
+                    [3, 3, 3, 3],  # 400->3, 500->3, 600->3, 1000->3
+                    [1, 2, 3, 3],  # 200->1, 300->2, 400->3, 500->3
+                    [2, 2, 3, 3],  # 300->2, 350->2, 400->3, 800->3
+                ],
+            ],
+            dtype='uint8',
+        )
+
+        image_palette = convert_colormap_to_palette(self.colormap)
+        actual_raster, _ = scale_paletted_1band(ds, image_palette)
+        assert_array_equal(expected_raster, actual_raster, strict=True)
+
+    def test_scale_paletted_1band_with_nan_and_clipping(self):
+        """Test that NaN values map to nodata while clipping still works."""
+        from hybig.browse import scale_paletted_1band
+
+        # Create test data with NaN, underflow, and overflow values
+        data_mixed = np.array(
+            [
+                [np.nan, -50, 100, 500],
+                [50, 200, 300, 1000],
+                [100, np.nan, 400, 600],
+                [-100, 250, np.nan, 800],
+            ]
+        ).astype('float64')
+        ds = DataArray(data_mixed).expand_dims('band')
+
+        # Expected: NaN -> 4 (nodata), underflow -> 0, overflow -> 3
+        expected_raster = np.array(
+            [
+                [
+                    [4, 0, 0, 3],  # NaN->4, -50->0, 100->0, 500->3
+                    [0, 1, 2, 3],  # 50->0, 200->1, 300->2, 1000->3
+                    [0, 4, 3, 3],  # 100->0, NaN->4, 400->3, 600->3
+                    [0, 1, 4, 3],  # -100->0, 250->1, NaN->4, 800->3
+                ],
+            ],
+            dtype='uint8',
+        )
+
+        image_palette = convert_colormap_to_palette(self.colormap)
+        actual_raster, actual_palette = scale_paletted_1band(ds, image_palette)
+        assert_array_equal(expected_raster, actual_raster, strict=True)
+
+        # Verify nodata color is transparent
+        expected_nodata_color = (0, 0, 0, 0)
+        self.assertEqual(actual_palette[np.uint8(4)], expected_nodata_color)
+
+    def test_scale_paletted_1band_to_rgb_clips_underflow_values(self):
+        """Test RGB output clips values below palette min to lowest color."""
+        from hybig.browse import scale_paletted_1band_to_rgb
+
+        # Create test data with values below the palette minimum
+        data_with_underflow = np.array(
+            [
+                [-50, 50, 100, 200],
+                [100, 200, 300, 400],
+            ]
+        ).astype('float64')
+        ds = DataArray(data_with_underflow).expand_dims('band')
+
+        image_palette = convert_colormap_to_palette(self.colormap)
+        actual_rgb, _ = scale_paletted_1band_to_rgb(ds, image_palette)
+
+        # Values -50 and 50 should get red color (255, 0, 0)
+        # which is the lowest color in the palette
+        self.assertEqual(actual_rgb[0, 0, 0], 255)  # Red channel for -50
+        self.assertEqual(actual_rgb[1, 0, 0], 0)  # Green channel for -50
+        self.assertEqual(actual_rgb[2, 0, 0], 0)  # Blue channel for -50
+
+        self.assertEqual(actual_rgb[0, 0, 1], 255)  # Red channel for 50
+        self.assertEqual(actual_rgb[1, 0, 1], 0)  # Green channel for 50
+        self.assertEqual(actual_rgb[2, 0, 1], 0)  # Blue channel for 50
+
+    def test_scale_paletted_1band_to_rgb_clips_overflow_values(self):
+        """Test RGB output clips values above palette max to highest color."""
+        from hybig.browse import scale_paletted_1band_to_rgb
+
+        # Create test data with values above the palette maximum
+        data_with_overflow = np.array(
+            [
+                [400, 500, 600, 1000],
+                [300, 400, 800, 1500],
+            ]
+        ).astype('float64')
+        ds = DataArray(data_with_overflow).expand_dims('band')
+
+        image_palette = convert_colormap_to_palette(self.colormap)
+        actual_rgb, _ = scale_paletted_1band_to_rgb(ds, image_palette)
+
+        # Values 500, 600, 1000, 800, 1500 should get blue color (0, 0, 255)
+        # which is the highest color in the palette
+        for col in [1, 2, 3]:  # columns 1, 2, 3 in row 0
+            self.assertEqual(actual_rgb[0, 0, col], 0)  # Red channel
+            self.assertEqual(actual_rgb[1, 0, col], 0)  # Green channel
+            self.assertEqual(actual_rgb[2, 0, col], 255)  # Blue channel
+
+        for col in [2, 3]:  # columns 2, 3 in row 1
+            self.assertEqual(actual_rgb[0, 1, col], 0)  # Red channel
+            self.assertEqual(actual_rgb[1, 1, col], 0)  # Green channel
+            self.assertEqual(actual_rgb[2, 1, col], 255)  # Blue channel
+
+    def test_scale_paletted_1band_to_rgb_with_nan_and_clipping(self):
+        """Test RGB output with NaN mapped to nodata and clipping working."""
+        from hybig.browse import scale_paletted_1band_to_rgb
+
+        # Create test data with NaN, underflow, and overflow values
+        data_mixed = np.array(
+            [
+                [np.nan, -50, 100, 500],
+                [50, 200, np.nan, 1000],
+            ]
+        ).astype('float64')
+        ds = DataArray(data_mixed).expand_dims('band')
+
+        image_palette = convert_colormap_to_palette(self.colormap)
+        actual_rgb, _ = scale_paletted_1band_to_rgb(ds, image_palette)
+
+        # NaN should map to nodata color (0, 0, 0)
+        self.assertEqual(actual_rgb[0, 0, 0], 0)  # Red for NaN at (0,0)
+        self.assertEqual(actual_rgb[1, 0, 0], 0)  # Green for NaN at (0,0)
+        self.assertEqual(actual_rgb[2, 0, 0], 0)  # Blue for NaN at (0,0)
+
+        self.assertEqual(actual_rgb[0, 1, 2], 0)  # Red for NaN at (1,2)
+        self.assertEqual(actual_rgb[1, 1, 2], 0)  # Green for NaN at (1,2)
+        self.assertEqual(actual_rgb[2, 1, 2], 0)  # Blue for NaN at (1,2)
+
+        # -50 and 50 should clip to red (255, 0, 0)
+        self.assertEqual(actual_rgb[0, 0, 1], 255)  # Red for -50
+        self.assertEqual(actual_rgb[0, 1, 0], 255)  # Red for 50
+
+        # 500 and 1000 should clip to blue (0, 0, 255)
+        self.assertEqual(actual_rgb[2, 0, 3], 255)  # Blue for 500
+        self.assertEqual(actual_rgb[2, 1, 3], 255)  # Blue for 1000
+
 
 class TestCreateBrowse(TestCase):
     """A class testing the create_browse function call.