From 57981b94f1b8ca0bece02e78d63f3f0f47e55b3c Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 11 Dec 2024 14:13:39 -0700 Subject: [PATCH 01/18] DAS-2276: Worked and committed on the wrong branch --- hybig/browse.py | 48 +++++++++++++++++++++++++++------------ tests/unit/test_browse.py | 38 ++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 16 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index 9af788f..ffa29b4 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -220,9 +220,11 @@ def create_browse_imagery( def convert_mulitband_to_raster(data_array: DataArray) -> ndarray: """Convert multiband to a raster image. - Reads the three or four bands from the file, then normalizes them to the range - 0 to 255. This assumes the input image is already in RGB or RGBA format and - just ensures that the output is 8bit. + Return a raster of a 4-band data set, the existing alpha layer is presumed to be the + missing data mask. + + Convert 3-band data into a 4-band raster by generating an alpha layer from + any missing data in the RGB bands. """ if data_array.rio.count not in [3, 4]: @@ -233,26 +235,42 @@ def convert_mulitband_to_raster(data_array: DataArray) -> ndarray: bands = data_array.to_numpy() - # Create an alpha layer where input NaN values are transparent. + if data_array.rio.count == 4: + return bands + + # Create a nan-based alpha layer where input NaN values are transparent. nan_mask = np.isnan(bands).any(axis=0) nan_alpha = np.where(nan_mask, TRANSPARENT, OPAQUE) - # grab any existing alpha layer - bands, image_alpha = remove_alpha(bands) + raster = convert_to_uint8(bands, data_array) + + return np.concatenate((raster, nan_alpha[None, ...]), axis=0) + - norm = Normalize(vmin=np.nanmin(bands), vmax=np.nanmax(bands)) - raster = np.nan_to_num(np.around(norm(bands) * 255.0), copy=False, nan=0.0).astype( - 'uint8' +def convert_to_uint8(bands: ndarray, data_array: DataArray) -> ndarray: + """Convert 3-band data with NaNs as missing values into uint8 data cube. + + 99.9% of the time this will simply pass through the data coercing it back + into unsigned int and setting the missing values to 0 that will be masked + as transparent in the output png. + + There is a non-zero, but very close to zero, chance that the input RGB + image was 16 bit and if any of the values exceed 255, we will normalize all + of input data to the range 0-255. + + """ + original_datatype = data_array.encoding.get('dtype') or data_array.encoding.get( + 'rasterio_dtype' ) - if image_alpha is not None: - # merge missing alpha with the image alpha band prefering transparency - # to opaqueness. - alpha = np.minimum(nan_alpha, image_alpha).astype(np.uint8) + if original_datatype != 'uint8' and np.nanmax(bands) > 255: + norm = Normalize(vmin=np.nanmin(bands), vmax=np.nanmax(bands)) + scaled = np.around(norm(bands) * 255.0) + raster = scaled.filled(0).astype('uint8') else: - alpha = nan_alpha + raster = np.nan_to_num(bands).astype('uint8') - return np.concatenate((raster, alpha[None, ...]), axis=0) + return raster def convert_singleband_to_raster( diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 6e25ac8..acb7f9b 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -421,7 +421,8 @@ def test_convert_singleband_to_raster_with_colormap_and_bad_data(self): actual_raster = convert_singleband_to_raster(ds, image_palette) assert_array_equal(expected_raster, actual_raster) - def test_convert_3_multiband_to_raster(self): + def test_convert_uint16_3_multiband_to_raster(self): + """Test that not uint8 input scales the output.""" bad_data = np.copy(self.data).astype('float64') bad_data[1][1] = np.nan bad_data[1][2] = np.nan @@ -429,6 +430,7 @@ def test_convert_3_multiband_to_raster(self): np.stack([self.data, bad_data, self.data]), dims=('band', 'y', 'x'), ) + ds.encoding = {'dtype': 'uint16'} expected_raster = np.array( [ @@ -463,6 +465,40 @@ def test_convert_3_multiband_to_raster(self): actual_raster = convert_mulitband_to_raster(ds) assert_array_equal(expected_raster, actual_raster.data) + def test_convert_uint8_3_multiband_to_raster(self): + """Ensure valid data is unchange when input is uint8.""" + scale_data = np.copy(self.data / 10).astype('float32') + + scale_data[1][1] = np.nan + scale_data[1][2] = np.nan + ds = DataArray( + np.stack([scale_data, scale_data, scale_data]), + dims=('band', 'y', 'x'), + ) + ds.encoding = {'dtype': 'uint8'} + + expected_data = scale_data.astype('uint8') + expected_data[1][1] = 0 + expected_data[1][2] = 0 + + expected_raster = np.array( + [ + expected_data, + expected_data, + expected_data, + [ + [OPAQUE, OPAQUE, OPAQUE, OPAQUE], + [OPAQUE, TRANSPARENT, TRANSPARENT, OPAQUE], + [OPAQUE, OPAQUE, OPAQUE, OPAQUE], + [OPAQUE, OPAQUE, OPAQUE, OPAQUE], + ], + ], + dtype='uint8', + ) + + actual_raster = convert_mulitband_to_raster(ds) + assert_array_equal(expected_raster, actual_raster.data) + def test_convert_4_multiband_to_raster(self): """Input data has NaN _fillValue match in the red layer at [1,1] and alpha channel also exists with a single transparent value at [0,0] From 94be7e55699aa3e5fe7a6b984dbe0fb97adcf720 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 11 Dec 2024 15:47:05 -0700 Subject: [PATCH 02/18] DAS-2276: convert_mulitband_to_raster only scales when needed 3 and 4 band input no longer scales by default. 4 bands are just converted to uint8 (scaling if necessary) 3 bands add an alpha layer where there are missing values. --- hybig/browse.py | 27 +++++--- hybig/sizes.py | 2 +- tests/unit/test_browse.py | 129 +++++++++++++++++++++++--------------- 3 files changed, 97 insertions(+), 61 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index ffa29b4..4919624 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -13,7 +13,7 @@ from harmony_service_lib.message import Source as HarmonySource from matplotlib.cm import ScalarMappable from matplotlib.colors import Normalize -from numpy import ndarray +from numpy import ndarray, uint8 from osgeo_utils.auxiliary.color_palette import ColorPalette from PIL import Image from rasterio.io import DatasetReader @@ -217,7 +217,7 @@ def create_browse_imagery( return processed_files -def convert_mulitband_to_raster(data_array: DataArray) -> ndarray: +def convert_mulitband_to_raster(data_array: DataArray) -> ndarray[uint8]: """Convert multiband to a raster image. Return a raster of a 4-band data set, the existing alpha layer is presumed to be the @@ -236,19 +236,19 @@ def convert_mulitband_to_raster(data_array: DataArray) -> ndarray: bands = data_array.to_numpy() if data_array.rio.count == 4: - return bands + return convert_to_uint8(bands, original_dtype(data_array)) # Create a nan-based alpha layer where input NaN values are transparent. nan_mask = np.isnan(bands).any(axis=0) nan_alpha = np.where(nan_mask, TRANSPARENT, OPAQUE) - raster = convert_to_uint8(bands, data_array) + raster = convert_to_uint8(bands, original_dtype(data_array)) return np.concatenate((raster, nan_alpha[None, ...]), axis=0) -def convert_to_uint8(bands: ndarray, data_array: DataArray) -> ndarray: - """Convert 3-band data with NaNs as missing values into uint8 data cube. +def convert_to_uint8(bands: ndarray, dtype: str | None) -> ndarray: + """Convert banded data with NaNs as missing values into uint8 data cube. 99.9% of the time this will simply pass through the data coercing it back into unsigned int and setting the missing values to 0 that will be masked @@ -259,11 +259,8 @@ def convert_to_uint8(bands: ndarray, data_array: DataArray) -> ndarray: of input data to the range 0-255. """ - original_datatype = data_array.encoding.get('dtype') or data_array.encoding.get( - 'rasterio_dtype' - ) - if original_datatype != 'uint8' and np.nanmax(bands) > 255: + if dtype != 'uint8' and np.nanmax(bands) > 255: norm = Normalize(vmin=np.nanmin(bands), vmax=np.nanmax(bands)) scaled = np.around(norm(bands) * 255.0) raster = scaled.filled(0).astype('uint8') @@ -273,6 +270,16 @@ def convert_to_uint8(bands: ndarray, data_array: DataArray) -> ndarray: return raster +def original_dtype(data_array: DataArray) -> str | None: + """Helper to return the original input data type. + + The input dtype is retained in the encoding dictionary and is used to know + what kind of casting is safe. + + """ + return data_array.encoding.get('dtype') or data_array.encoding.get('rasterio_dtype') + + def convert_singleband_to_raster( data_array: DataArray, color_palette: ColorPalette | None = None, diff --git a/hybig/sizes.py b/hybig/sizes.py index f1d58f0..c2fd2e7 100644 --- a/hybig/sizes.py +++ b/hybig/sizes.py @@ -422,7 +422,7 @@ def find_closest_resolution( """ best_info = None - smallest_diff = np.Infinity + smallest_diff = np.inf for res in resolutions: for info in resolution_info: resolution_diff = np.abs(res - info.pixel_size) diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index acb7f9b..43e8b8a 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -499,61 +499,70 @@ def test_convert_uint8_3_multiband_to_raster(self): actual_raster = convert_mulitband_to_raster(ds) assert_array_equal(expected_raster, actual_raster.data) - def test_convert_4_multiband_to_raster(self): - """Input data has NaN _fillValue match in the red layer at [1,1] - and alpha channel also exists with a single transparent value at [0,0] + def test_convert_4_multiband_uint8_to_raster(self): + """4-band 'uint8' images are returned unchanged.""" + ds = Mock(DataArray) + ds.rio.count = 4 + + r_data = np.array( + [ + [10, 200, 30, 40], + [10, 200, 30, 40], + [10, 200, 30, 40], + [10, 200, 30, 40], + ] + ).astype('uint8') - See that the expected output has transformed the missing data [nan] - into fully transparent at [1,1] and retained the transparent value of 1 - at [0,0] + g_data = r_data.copy() + b_data = r_data.copy() - """ - ds = Mock(DataArray) - bad_data = np.copy(self.data).astype('float64') - bad_data[1, 1] = np.nan + a_data = np.ones_like(self.data) * 255 + a_data[0, 0] = 0 - alpha = np.ones_like(self.data) * 255 - alpha[0, 0] = 1 + to_numpy_result = np.stack([r_data, g_data, b_data, a_data]) + + ds.to_numpy.return_value = to_numpy_result + + expected_raster = to_numpy_result + + actual_raster = convert_mulitband_to_raster(ds) + assert_array_equal(expected_raster, actual_raster.data) + + def test_convert_4_multiband_uint16_to_raster(self): + """4-band 'uint16' images are scaled if the range exceeds 255.""" + ds = Mock(DataArray) ds.rio.count = 4 - ds.to_numpy.return_value = np.stack([bad_data, self.data, self.data, alpha]) - expected_raster = np.array( + + r_data = np.array( [ - [ - [0, 85, 170, 255], - [0, 0, 170, 255], - [0, 85, 170, 255], - [0, 85, 170, 255], - ], - [ - [0, 85, 170, 255], - [0, 85, 170, 255], - [0, 85, 170, 255], - [0, 85, 170, 255], - ], - [ - [0, 85, 170, 255], - [0, 85, 170, 255], - [0, 85, 170, 255], - [0, 85, 170, 255], - ], - [ - [1, 255, 255, 255], - [255, 0, 255, 255], - [255, 255, 255, 255], - [255, 255, 255, 255], - ], - ], - dtype='uint8', - ) + [10, 200, 300, 400], + [10, 200, 300, 400], + [10, 200, 300, 400], + [10, 200, 300, 400], + ] + ).astype('uint16') + g_data = r_data.copy() + b_data = r_data.copy() + + a_data = np.ones_like(self.data) * 255 + a_data[0, 0] = 0 + + to_numpy_result = np.stack([r_data, g_data, b_data, a_data]) + + ds.to_numpy.return_value = to_numpy_result + + # expect the input data to have 0 to 400 to be scaled into 0 to 255 + expected_raster = np.around( + np.interp(to_numpy_result, (0, 400), (0.0, 1.0)) * 255.0 + ).astype('uint8') actual_raster = convert_mulitband_to_raster(ds) assert_array_equal(expected_raster, actual_raster.data) def test_convert_4_multiband_masked_to_raster(self): - """Input data is selected from a subset of a real OPERA RTC input data - file that has masked the alpha layer and as a result as a datatype of - float32. + """4-band images are returned as uint8s** + **Even if they are masked, but also nans are converted to 0s. """ ds = Mock(DataArray) ds.rio.count = 4 @@ -589,12 +598,32 @@ def test_convert_4_multiband_masked_to_raster(self): ) ds.to_numpy.return_value = input_array - expected_raster = np.ma.array( - data=[ - [[0, 0, 0, 121], [0, 0, 0, 64], [0, 0, 255, 0], [0, 0, 13, 255]], - [[0, 0, 0, 255], [0, 0, 0, 255], [0, 0, 255, 255], [0, 0, 255, 255]], - [[0, 0, 0, 121], [0, 0, 0, 64], [0, 0, 255, 0], [0, 0, 13, 255]], - [[0, 0, 0, 255], [0, 0, 0, 255], [0, 0, 255, 255], [0, 0, 255, 255]], + expected_raster = np.array( + [ + [ + [0, 0, 0, 234], + [0, 0, 0, 225], + [0, 0, 255, 215], + [0, 0, 217, 255], + ], + [ + [0, 0, 0, 255], + [0, 0, 0, 255], + [0, 0, 255, 255], + [0, 0, 255, 255], + ], + [ + [0, 0, 0, 234], + [0, 0, 0, 225], + [0, 0, 255, 215], + [0, 0, 217, 255], + ], + [ + [0, 0, 0, 255], + [0, 0, 0, 255], + [0, 0, 255, 255], + [0, 0, 255, 255], + ], ], dtype=np.uint8, ) From dd60d72095a97e1133e3d4b2acaa7e907f3816af Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 11 Dec 2024 16:03:51 -0700 Subject: [PATCH 03/18] DAS-2276: Rewords comments. --- hybig/browse.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index 4919624..56212d0 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -247,16 +247,16 @@ def convert_mulitband_to_raster(data_array: DataArray) -> ndarray[uint8]: return np.concatenate((raster, nan_alpha[None, ...]), axis=0) -def convert_to_uint8(bands: ndarray, dtype: str | None) -> ndarray: - """Convert banded data with NaNs as missing values into uint8 data cube. +def convert_to_uint8(bands: ndarray, dtype: str | None) -> ndarray[uint8]: + """Convert Banded data with NaNs (missing) into a uint8 data cube. - 99.9% of the time this will simply pass through the data coercing it back - into unsigned int and setting the missing values to 0 that will be masked + 99.99% of the time this will simply pass through the data coercing it back + into unsigned ints and setting the missing values to 0 that will be masked as transparent in the output png. - There is a non-zero, but very close to zero, chance that the input RGB - image was 16 bit and if any of the values exceed 255, we will normalize all - of input data to the range 0-255. + There is a some small non-zero chance that the input RGB image was 16-bit + and if any of the values exceed 255, we must normalize all of input data to + the range 0-255. """ @@ -271,10 +271,10 @@ def convert_to_uint8(bands: ndarray, dtype: str | None) -> ndarray: def original_dtype(data_array: DataArray) -> str | None: - """Helper to return the original input data type. + """Return the original input data's type. - The input dtype is retained in the encoding dictionary and is used to know - what kind of casting is safe. + The input dtype is retained in the encoding dictionary and can be used to + understand what kind of casts are safe. """ return data_array.encoding.get('dtype') or data_array.encoding.get('rasterio_dtype') From 8e67313dba68b2c88cf1650af834507359d4375f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 11 Dec 2024 16:14:30 -0700 Subject: [PATCH 04/18] DAS-2276: Compare array types in assertions. --- tests/test_service/test_adapter.py | 16 ++++++--- tests/unit/test_browse.py | 54 +++++++++++++++++------------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/tests/test_service/test_adapter.py b/tests/test_service/test_adapter.py index 3e30b7b..f60e802 100644 --- a/tests/test_service/test_adapter.py +++ b/tests/test_service/test_adapter.py @@ -270,10 +270,14 @@ def move_tif(*args, **kwargs): mock_reproject.call_args_list, expected_reproject_calls ): np.testing.assert_array_equal( - actual_call.kwargs['source'], expected_call.kwargs['source'] + actual_call.kwargs['source'], + expected_call.kwargs['source'], + strict=True, ) np.testing.assert_array_equal( - actual_call.kwargs['destination'], expected_call.kwargs['destination'] + actual_call.kwargs['destination'], + expected_call.kwargs['destination'], + strict=True, ) self.assertEqual( actual_call.kwargs['src_transform'], @@ -482,10 +486,14 @@ def move_tif(*args, **kwargs): mock_reproject.call_args_list, expected_reproject_calls ): np.testing.assert_array_equal( - actual_call.kwargs['source'], expected_call.kwargs['source'] + actual_call.kwargs['source'], + expected_call.kwargs['source'], + strict=True, ) np.testing.assert_array_equal( - actual_call.kwargs['destination'], expected_call.kwargs['destination'] + actual_call.kwargs['destination'], + expected_call.kwargs['destination'], + strict=True, ) self.assertEqual( actual_call.kwargs['src_transform'], diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 43e8b8a..1d35725 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -266,10 +266,14 @@ def test_create_browse_imagery_with_mocks( reproject_mock.call_args_list, expected_calls ): np.testing.assert_array_equal( - actual_call.kwargs['source'], expected_call.kwargs['source'] + actual_call.kwargs['source'], + expected_call.kwargs['source'], + strict=True, ) np.testing.assert_array_equal( - actual_call.kwargs['destination'], expected_call.kwargs['destination'] + actual_call.kwargs['destination'], + expected_call.kwargs['destination'], + strict=True, ) self.assertEqual( actual_call.kwargs['src_transform'], @@ -339,7 +343,7 @@ def test_convert_singleband_to_raster_without_colortable(self): dtype='uint8', ) actual_raster = convert_singleband_to_raster(ds, None) - assert_array_equal(expected_raster, actual_raster) + assert_array_equal(expected_raster, actual_raster, strict=True) def test_convert_singleband_to_raster_with_colormap(self): ds = DataArray(self.data).expand_dims('band') @@ -376,7 +380,7 @@ def test_convert_singleband_to_raster_with_colormap(self): # Read down: red, yellow, green, blue image_palette = convert_colormap_to_palette(self.colormap) actual_raster = convert_singleband_to_raster(ds, image_palette) - assert_array_equal(expected_raster, actual_raster) + assert_array_equal(expected_raster, actual_raster, strict=True) def test_convert_singleband_to_raster_with_colormap_and_bad_data(self): data_array = np.array(self.data, dtype='float') @@ -419,10 +423,10 @@ def test_convert_singleband_to_raster_with_colormap_and_bad_data(self): image_palette = convert_colormap_to_palette(colormap) actual_raster = convert_singleband_to_raster(ds, image_palette) - assert_array_equal(expected_raster, actual_raster) + assert_array_equal(expected_raster, actual_raster, strict=True) def test_convert_uint16_3_multiband_to_raster(self): - """Test that not uint8 input scales the output.""" + """Test that uint16 input scales the output.""" bad_data = np.copy(self.data).astype('float64') bad_data[1][1] = np.nan bad_data[1][2] = np.nan @@ -463,21 +467,26 @@ def test_convert_uint16_3_multiband_to_raster(self): ) actual_raster = convert_mulitband_to_raster(ds) - assert_array_equal(expected_raster, actual_raster.data) + assert_array_equal(expected_raster, actual_raster.data, strict=True) def test_convert_uint8_3_multiband_to_raster(self): """Ensure valid data is unchange when input is uint8.""" - scale_data = np.copy(self.data / 10).astype('float32') + scale_data = np.array( + [ + [10, 200, 30, 40], + [10, np.nan, np.nan, 40], + [10, 200, 30, 40], + [10, 200, 30, 40], + ] + ).astype('float32') - scale_data[1][1] = np.nan - scale_data[1][2] = np.nan ds = DataArray( np.stack([scale_data, scale_data, scale_data]), dims=('band', 'y', 'x'), ) ds.encoding = {'dtype': 'uint8'} - expected_data = scale_data.astype('uint8') + expected_data = scale_data.copy() expected_data[1][1] = 0 expected_data[1][2] = 0 @@ -497,7 +506,7 @@ def test_convert_uint8_3_multiband_to_raster(self): ) actual_raster = convert_mulitband_to_raster(ds) - assert_array_equal(expected_raster, actual_raster.data) + assert_array_equal(expected_raster, actual_raster.data, strict=True) def test_convert_4_multiband_uint8_to_raster(self): """4-band 'uint8' images are returned unchanged.""" @@ -523,10 +532,10 @@ def test_convert_4_multiband_uint8_to_raster(self): ds.to_numpy.return_value = to_numpy_result - expected_raster = to_numpy_result + expected_raster = to_numpy_result.astype('uint8') actual_raster = convert_mulitband_to_raster(ds) - assert_array_equal(expected_raster, actual_raster.data) + assert_array_equal(expected_raster, actual_raster.data, strict=True) def test_convert_4_multiband_uint16_to_raster(self): """4-band 'uint16' images are scaled if the range exceeds 255.""" @@ -557,13 +566,10 @@ def test_convert_4_multiband_uint16_to_raster(self): ).astype('uint8') actual_raster = convert_mulitband_to_raster(ds) - assert_array_equal(expected_raster, actual_raster.data) + assert_array_equal(expected_raster, actual_raster.data, strict=True) def test_convert_4_multiband_masked_to_raster(self): - """4-band images are returned as uint8s** - - **Even if they are masked, but also nans are converted to 0s. - """ + """4-band images are returned with nan -> 0""" ds = Mock(DataArray) ds.rio.count = 4 nan = np.nan @@ -629,7 +635,7 @@ def test_convert_4_multiband_masked_to_raster(self): ) actual_raster = convert_mulitband_to_raster(ds) - assert_array_equal(expected_raster.data, actual_raster.data) + assert_array_equal(expected_raster.data, actual_raster.data, strict=True) def test_convert_5_multiband_to_raster(self): ds = Mock(DataArray) @@ -654,7 +660,7 @@ def test_prepare_raster_for_writing_jpeg_3band(self): actual_raster, actual_color_map = prepare_raster_for_writing(raster, driver) self.assertEqual(expected_color_map, actual_color_map) - np.testing.assert_array_equal(expected_raster, actual_raster) + np.testing.assert_array_equal(expected_raster, actual_raster, strict=True) def test_prepare_raster_for_writing_jpeg_4band(self): raster = self.random.integers(255, size=(4, 7, 8)) @@ -663,7 +669,7 @@ def test_prepare_raster_for_writing_jpeg_4band(self): expected_color_map = None actual_raster, actual_color_map = prepare_raster_for_writing(raster, driver) self.assertEqual(expected_color_map, actual_color_map) - np.testing.assert_array_equal(expected_raster, actual_raster) + np.testing.assert_array_equal(expected_raster, actual_raster, strict=True) @patch('hybig.browse.palettize_raster') def test_prepare_raster_for_writing_png_4band(self, palettize_mock): @@ -694,7 +700,7 @@ def test_palettize_raster_no_alpha_layer(self, get_color_map_mock, image_mock): multiband_image_mock.quantize.assert_called_once_with(colors=254) get_color_map_mock.assert_called_once_with(quantized_output) - np.testing.assert_array_equal(expected_out_raster, out_raster) + np.testing.assert_array_equal(expected_out_raster, out_raster, strict=True) @patch('hybig.browse.Image') @patch('hybig.browse.get_color_map_from_image') @@ -722,7 +728,7 @@ def test_palettize_raster_with_alpha_layer(self, get_color_map_mock, image_mock) multiband_image_mock.quantize.assert_called_once_with(colors=254) get_color_map_mock.assert_called_once_with(quantized_output) - np.testing.assert_array_equal(expected_out_raster, out_raster) + np.testing.assert_array_equal(expected_out_raster, out_raster, strict=True) def test_get_color_map_from_image(self): """PIL Image yields a color_map From bbd1891a049b6ce066188b6aaac3c0e8b043a9ca Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 11 Dec 2024 16:39:49 -0700 Subject: [PATCH 05/18] DAS-2276: Quick test --- hybig/browse.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hybig/browse.py b/hybig/browse.py index 56212d0..b84416c 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -181,7 +181,11 @@ def create_browse_imagery( f'incorrect number of bands for image: {rio_in_array.rio.count}' ) - raster, color_map = prepare_raster_for_writing(raster, output_driver) + if rio_in_array.rio.count == 1: + # we only paletize single band input data + raster, color_map = prepare_raster_for_writing(raster, output_driver) + else: + color_map = None grid_parameters = get_target_grid_parameters(message, rio_in_array) grid_parameter_list, tile_locators = create_tiled_output_parameters( From 97086e59e54e6800871570e4a4c99a2727fd6445 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Thu, 12 Dec 2024 12:15:23 -0700 Subject: [PATCH 06/18] DAS-2276: Use OPAQUE and TRANSPARENT. Reword comments. --- hybig/browse.py | 10 +++++----- tests/unit/test_browse.py | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index b84416c..bb152db 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -224,8 +224,8 @@ def create_browse_imagery( def convert_mulitband_to_raster(data_array: DataArray) -> ndarray[uint8]: """Convert multiband to a raster image. - Return a raster of a 4-band data set, the existing alpha layer is presumed to be the - missing data mask. + Return a 4-band raster, where the alpha layer is presumed to be the missing + data mask. Convert 3-band data into a 4-band raster by generating an alpha layer from any missing data in the RGB bands. @@ -242,7 +242,7 @@ def convert_mulitband_to_raster(data_array: DataArray) -> ndarray[uint8]: if data_array.rio.count == 4: return convert_to_uint8(bands, original_dtype(data_array)) - # Create a nan-based alpha layer where input NaN values are transparent. + # Input NaNs in any of the RGB bands are made transparent. nan_mask = np.isnan(bands).any(axis=0) nan_alpha = np.where(nan_mask, TRANSPARENT, OPAQUE) @@ -277,8 +277,8 @@ def convert_to_uint8(bands: ndarray, dtype: str | None) -> ndarray[uint8]: def original_dtype(data_array: DataArray) -> str | None: """Return the original input data's type. - The input dtype is retained in the encoding dictionary and can be used to - understand what kind of casts are safe. + rastero_optn retains the input dtype in the encoding dictionary and is used + to understand what kind of casts are safe. """ return data_array.encoding.get('dtype') or data_array.encoding.get('rasterio_dtype') diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 1d35725..d844fba 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -553,14 +553,15 @@ def test_convert_4_multiband_uint16_to_raster(self): g_data = r_data.copy() b_data = r_data.copy() - a_data = np.ones_like(self.data) * 255 - a_data[0, 0] = 0 + a_data = np.ones_like(self.data) * OPAQUE + a_data[0, 0] = TRANSPARENT to_numpy_result = np.stack([r_data, g_data, b_data, a_data]) ds.to_numpy.return_value = to_numpy_result - # expect the input data to have 0 to 400 to be scaled into 0 to 255 + # expect the input data to have the data values from 0 to 400 to be + # scaled into the range 0 to 255. expected_raster = np.around( np.interp(to_numpy_result, (0, 400), (0.0, 1.0)) * 255.0 ).astype('uint8') From b17e0202eb94fd7f7a52d0fcf85178c9ff7c898c Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 13 Dec 2024 10:13:22 -0700 Subject: [PATCH 07/18] DAS-2276: Fix browse code for jpeg images. --- hybig/browse.py | 17 +++++++---------- tests/test_service/test_adapter.py | 5 +++-- tests/unit/test_browse.py | 11 +++++++++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index bb152db..f09b4f9 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -181,11 +181,11 @@ def create_browse_imagery( f'incorrect number of bands for image: {rio_in_array.rio.count}' ) - if rio_in_array.rio.count == 1: + raster, color_map = prepare_raster_for_writing(raster, output_driver) + + if rio_in_array.rio.count == 1 and output_driver == 'PNG': # we only paletize single band input data - raster, color_map = prepare_raster_for_writing(raster, output_driver) - else: - color_map = None + raster, color_map = palettize_raster(raster) grid_parameters = get_target_grid_parameters(message, rio_in_array) grid_parameter_list, tile_locators = create_tiled_output_parameters( @@ -363,12 +363,9 @@ def prepare_raster_for_writing( raster: ndarray, driver: str ) -> tuple[ndarray, dict | None]: """Remove alpha layer if writing a jpeg.""" - if driver == 'JPEG': - if raster.shape[0] == 4: - raster = raster[0:3, :, :] - return raster, None - - return palettize_raster(raster) + if driver == 'JPEG' and raster.shape[0] == 4: + raster = raster[0:3, :, :] + return raster, None def palettize_raster(raster: ndarray) -> tuple[ndarray, dict]: diff --git a/tests/test_service/test_adapter.py b/tests/test_service/test_adapter.py index f60e802..99e2d72 100644 --- a/tests/test_service/test_adapter.py +++ b/tests/test_service/test_adapter.py @@ -470,7 +470,7 @@ def move_tif(*args, **kwargs): expected_reproject_calls = [ call( - source=raster[0, :, :], + source=raster[band, :, :], destination=dest, src_transform=rio_data_array.rio.transform(), src_crs=rio_data_array.rio.crs, @@ -479,9 +479,10 @@ def move_tif(*args, **kwargs): dst_nodata=255, resampling=Resampling.nearest, ) + for band in range(4) ] - self.assertEqual(mock_reproject.call_count, 1) + self.assertEqual(mock_reproject.call_count, 4) for actual_call, expected_call in zip( mock_reproject.call_args_list, expected_reproject_calls ): diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index d844fba..615b69d 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -677,9 +677,16 @@ def test_prepare_raster_for_writing_png_4band(self, palettize_mock): raster = self.random.integers(255, size=(4, 7, 8)) driver = 'PNG' - prepare_raster_for_writing(raster, driver) + expected, _ = prepare_raster_for_writing(raster, driver) + np.testing.assert_array_equal(raster, expected, strict=True) - palettize_mock.assert_called_once_with(raster) + @patch('hybig.browse.palettize_raster') + def test_prepare_raster_for_writing_png_3band(self, palettize_mock): + raster = self.random.integers(255, size=(3, 7, 8)) + driver = 'PNG' + + expected, _ = prepare_raster_for_writing(raster, driver) + np.testing.assert_array_equal(raster, expected, strict=True) @patch('hybig.browse.Image') @patch('hybig.browse.get_color_map_from_image') From e229a8f712a31f1305b6f1147916adb3cdc37341 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 13 Dec 2024 11:24:06 -0700 Subject: [PATCH 08/18] DAS-2276: Refactor prepare_raster_for_writing. Moves the palettize_raster function inside again. --- hybig/browse.py | 21 ++++++++++++++------- tests/test_service/test_adapter.py | 2 +- tests/unit/test_browse.py | 27 +++++++++++++++++++++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index f09b4f9..1fb1266 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -181,11 +181,9 @@ def create_browse_imagery( f'incorrect number of bands for image: {rio_in_array.rio.count}' ) - raster, color_map = prepare_raster_for_writing(raster, output_driver) - - if rio_in_array.rio.count == 1 and output_driver == 'PNG': - # we only paletize single band input data - raster, color_map = palettize_raster(raster) + raster, color_map = prepare_raster_for_writing( + raster, output_driver, rio_in_array.rio.count + ) grid_parameters = get_target_grid_parameters(message, rio_in_array) grid_parameter_list, tile_locators = create_tiled_output_parameters( @@ -360,12 +358,21 @@ def image_driver(mime: str) -> str: def prepare_raster_for_writing( - raster: ndarray, driver: str + raster: ndarray, + driver: str, + input_bands: int, ) -> tuple[ndarray, dict | None]: """Remove alpha layer if writing a jpeg.""" + color_map = None if driver == 'JPEG' and raster.shape[0] == 4: raster = raster[0:3, :, :] - return raster, None + return raster, color_map + + if input_bands == 1 and driver == 'PNG': + # we only paletize single band input data + raster, color_map = palettize_raster(raster) + + return raster, color_map def palettize_raster(raster: ndarray) -> tuple[ndarray, dict]: diff --git a/tests/test_service/test_adapter.py b/tests/test_service/test_adapter.py index 99e2d72..b3bc5cf 100644 --- a/tests/test_service/test_adapter.py +++ b/tests/test_service/test_adapter.py @@ -460,7 +460,7 @@ def move_tif(*args, **kwargs): 'count': 3, } raster = convert_mulitband_to_raster(rio_data_array) - raster, color_map = prepare_raster_for_writing(raster, 'PNG') + raster, color_map = prepare_raster_for_writing(raster, 'PNG', 3) dest = np.full( (expected_params['height'], expected_params['width']), diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 615b69d..c083d92 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -655,20 +655,26 @@ def test_convert_5_multiband_to_raster(self): def test_prepare_raster_for_writing_jpeg_3band(self): raster = self.random.integers(255, size=(3, 5, 6)) + count = 'irrelevant' driver = 'JPEG' expected_raster = np.copy(raster) expected_color_map = None - actual_raster, actual_color_map = prepare_raster_for_writing(raster, driver) + actual_raster, actual_color_map = prepare_raster_for_writing( + raster, driver, count + ) self.assertEqual(expected_color_map, actual_color_map) np.testing.assert_array_equal(expected_raster, actual_raster, strict=True) def test_prepare_raster_for_writing_jpeg_4band(self): raster = self.random.integers(255, size=(4, 7, 8)) driver = 'JPEG' + count = 'irrelevant' expected_raster = np.copy(raster[0:3, :, :]) expected_color_map = None - actual_raster, actual_color_map = prepare_raster_for_writing(raster, driver) + actual_raster, actual_color_map = prepare_raster_for_writing( + raster, driver, count + ) self.assertEqual(expected_color_map, actual_color_map) np.testing.assert_array_equal(expected_raster, actual_raster, strict=True) @@ -676,17 +682,30 @@ def test_prepare_raster_for_writing_jpeg_4band(self): def test_prepare_raster_for_writing_png_4band(self, palettize_mock): raster = self.random.integers(255, size=(4, 7, 8)) driver = 'PNG' + count = 'not 1' - expected, _ = prepare_raster_for_writing(raster, driver) + expected, _ = prepare_raster_for_writing(raster, driver, count) np.testing.assert_array_equal(raster, expected, strict=True) + palettize_mock.assert_not_called() @patch('hybig.browse.palettize_raster') def test_prepare_raster_for_writing_png_3band(self, palettize_mock): raster = self.random.integers(255, size=(3, 7, 8)) driver = 'PNG' + count = 'not 1' - expected, _ = prepare_raster_for_writing(raster, driver) + expected, _ = prepare_raster_for_writing(raster, driver, count) np.testing.assert_array_equal(raster, expected, strict=True) + palettize_mock.assert_not_called() + + @patch('hybig.browse.palettize_raster') + def test_prepare_1band_raster_for_writing_png(self, palettize_mock): + raster = self.random.integers(255, size=(1, 7, 8)) + driver = 'PNG' + count = 1 + palettize_mock.return_value = (None, None) + expected, _ = prepare_raster_for_writing(raster, driver, count) + palettize_mock.assert_called_with(raster) @patch('hybig.browse.Image') @patch('hybig.browse.get_color_map_from_image') From 664e439757046a12f3e8b225d1432c6479a90e2e Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 13 Dec 2024 11:41:57 -0700 Subject: [PATCH 09/18] DAS-2276: Update Changelog and bump vervice version number. --- CHANGELOG.md | 6 ++++-- docker/service_version.txt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53a76e7..f519700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ HyBIG follows semantic versioning. All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). -## [unreleased] - 2024-12-10 +## [v2.1.0] - 2024-12-13 ### Changed +* Input GeoTIFF RGB[A] images are **no longer palettized** when converted to a PNG. The new resulting output browse images are now 3 or 4 band PNG retaining the color information of the input image.[#39](https://github.com/nasa/harmony-browse-image-generator/pull/39) * Changed pre-commit configuration to remove `black-jupyter` dependency [#38](https://github.com/nasa/harmony-browse-image-generator/pull/38) * Updates service image's python to 3.12 [#38](https://github.com/nasa/harmony-browse-image-generator/pull/38) * Simplifies test scripts to run with pytest and pytest plugins [#38](https://github.com/nasa/harmony-browse-image-generator/pull/38) @@ -90,7 +91,8 @@ outlined by the NASA open-source guidelines. For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. -[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.2..HEAD +[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/2.1.0..HEAD +[v2.1.0]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.2..2.1.0 [v2.0.2]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.1..2.0.2 [v2.0.1]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.0..2.0.1 [v2.0.0]:https://github.com/nasa/harmony-browse-image-generator/compare/1.2.2..2.0.0 diff --git a/docker/service_version.txt b/docker/service_version.txt index e9307ca..7ec1d6d 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -2.0.2 +2.1.0 From 32fb4db9ca4dc6e0322a464d23754b5884282093 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 13 Dec 2024 11:55:10 -0700 Subject: [PATCH 10/18] DAS-2276: Simplify prepare_raster_for_writing. --- hybig/browse.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index 1fb1266..d760cbf 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -363,16 +363,14 @@ def prepare_raster_for_writing( input_bands: int, ) -> tuple[ndarray, dict | None]: """Remove alpha layer if writing a jpeg.""" - color_map = None if driver == 'JPEG' and raster.shape[0] == 4: - raster = raster[0:3, :, :] - return raster, color_map + return raster[0:3, :, :], None - if input_bands == 1 and driver == 'PNG': + if driver == 'PNG' and input_bands == 1: # we only paletize single band input data - raster, color_map = palettize_raster(raster) + return palettize_raster(raster) - return raster, color_map + return raster, None def palettize_raster(raster: ndarray) -> tuple[ndarray, dict]: From 25742bf94c3dbd547af0789f6f63470a83fa0774 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 13 Dec 2024 11:58:39 -0700 Subject: [PATCH 11/18] DAS-2276: better example: test_convert_uint8_3_multiband_to_raster --- tests/unit/test_browse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index c083d92..667ea5c 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -470,7 +470,7 @@ def test_convert_uint16_3_multiband_to_raster(self): assert_array_equal(expected_raster, actual_raster.data, strict=True) def test_convert_uint8_3_multiband_to_raster(self): - """Ensure valid data is unchange when input is uint8.""" + """Ensure valid data is unchanged when input is uint8.""" scale_data = np.array( [ [10, 200, 30, 40], @@ -525,20 +525,20 @@ def test_convert_4_multiband_uint8_to_raster(self): g_data = r_data.copy() b_data = r_data.copy() - a_data = np.ones_like(self.data) * 255 + a_data = np.ones_like(r_data) * 255 a_data[0, 0] = 0 to_numpy_result = np.stack([r_data, g_data, b_data, a_data]) ds.to_numpy.return_value = to_numpy_result - expected_raster = to_numpy_result.astype('uint8') + expected_raster = to_numpy_result actual_raster = convert_mulitband_to_raster(ds) assert_array_equal(expected_raster, actual_raster.data, strict=True) def test_convert_4_multiband_uint16_to_raster(self): - """4-band 'uint16' images are scaled if the range exceeds 255.""" + """4-band 'uint16' images are scaled if their range exceeds 255.""" ds = Mock(DataArray) ds.rio.count = 4 From dd098155947ce952f4e8637c8863cc4739aef803 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 16 Dec 2024 09:57:43 -0700 Subject: [PATCH 12/18] DAS-2276: Write transparent backgrounds when regrid dest has nodata fix previous assumption that the 255 value is set to an RGBA value for transparency. Now we are writing rgba without the quantization. --- hybig/browse.py | 6 +++++- tests/test_service/test_adapter.py | 4 ++-- tests/unit/test_browse.py | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index d760cbf..89d9256 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -507,9 +507,13 @@ def write_georaster_as_browse( """ n_bands = raster.shape[0] - dst_nodata = NODATA_IDX + if color_map is not None: + dst_nodata = NODATA_IDX color_map[dst_nodata] = NODATA_RGBA + else: + # for banded data set the each band's destination nodata to zero (TRANSPARENT). + dst_nodata = TRANSPARENT creation_options = { **grid_parameters, diff --git a/tests/test_service/test_adapter.py b/tests/test_service/test_adapter.py index b3bc5cf..f4e87ff 100644 --- a/tests/test_service/test_adapter.py +++ b/tests/test_service/test_adapter.py @@ -456,7 +456,7 @@ def move_tif(*args, **kwargs): 'transform': expected_transform, 'driver': 'PNG', 'dtype': 'uint8', - 'dst_nodata': 255, + 'dst_nodata': 0, 'count': 3, } raster = convert_mulitband_to_raster(rio_data_array) @@ -476,7 +476,7 @@ def move_tif(*args, **kwargs): src_crs=rio_data_array.rio.crs, dst_transform=expected_params['transform'], dst_crs=expected_params['crs'], - dst_nodata=255, + dst_nodata=expected_params['dst_nodata'], resampling=Resampling.nearest, ) for band in range(4) diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 667ea5c..dc89feb 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -237,7 +237,7 @@ def test_create_browse_imagery_with_mocks( src_crs=da_mock.rio.crs, dst_transform=target_transform, dst_crs=CRS.from_string('EPSG:4326'), - dst_nodata=255, + dst_nodata=0, resampling=Resampling.nearest, ), call( @@ -247,7 +247,7 @@ def test_create_browse_imagery_with_mocks( src_crs=da_mock.rio.crs, dst_transform=target_transform, dst_crs=CRS.from_string('EPSG:4326'), - dst_nodata=255, + dst_nodata=0, resampling=Resampling.nearest, ), call( @@ -257,7 +257,7 @@ def test_create_browse_imagery_with_mocks( src_crs=da_mock.rio.crs, dst_transform=target_transform, dst_crs=CRS.from_string('EPSG:4326'), - dst_nodata=255, + dst_nodata=0, resampling=Resampling.nearest, ), ] From 3864f030e054f6ca3972ef7cd3929be1483700db Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 17 Dec 2024 15:44:10 -0700 Subject: [PATCH 13/18] DAS-2276: Fixup TAGS in changelog --- CHANGELOG.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f519700..1ce6c2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,15 +91,15 @@ outlined by the NASA open-source guidelines. For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. -[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/2.1.0..HEAD -[v2.1.0]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.2..2.1.0 -[v2.0.2]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.1..2.0.2 -[v2.0.1]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.0..2.0.1 -[v2.0.0]:https://github.com/nasa/harmony-browse-image-generator/compare/1.2.2..2.0.0 -[v1.2.2]: https://github.com/nasa/harmony-browse-image-generator/compare/1.2.1..1.2.2 -[v1.2.1]: https://github.com/nasa/harmony-browse-image-generator/compare/1.2.0..1.2.1 -[v1.2.0]: https://github.com/nasa/harmony-browse-image-generator/compare/1.1.0..1.2.0 -[v1.1.0]: https://github.com/nasa/harmony-browse-image-generator/compare/1.0.2..1.1.0 -[v1.0.2]: https://github.com/nasa/harmony-browse-image-generator/compare/1.0.1..1.0.2 -[v1.0.1]: https://github.com/nasa/harmony-browse-image-generator/compare/1.0.0..1.0.1 -[v1.0.0]: https://github.com/nasa/harmony-browse-image-generator/compare/0.0.11-legacy..1.0.0 +[unreleased]: https://github.com/nasa/harmony-browse-image-generator/ +[v2.1.0]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/2.1.0 +[v2.0.2]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/2.0.2 +[v2.0.1]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/2.0.1 +[v2.0.0]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/2.0.0 +[v1.2.2]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/1.2.2 +[v1.2.1]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/1.2.1 +[v1.2.0]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/1.2.0 +[v1.1.0]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/1.1.0 +[v1.0.2]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/1.0.2 +[v1.0.1]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/1.0.1 +[v1.0.0]: https://github.com/nasa/harmony-browse-image-generator/releases/tag/1.0.0 From 8d8968c57c5f4017a287bd1524d8c7fd0de308ac Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 17 Dec 2024 15:46:16 -0700 Subject: [PATCH 14/18] DAS-2276: reword comment. --- hybig/browse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index 89d9256..b709955 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -252,9 +252,9 @@ def convert_mulitband_to_raster(data_array: DataArray) -> ndarray[uint8]: def convert_to_uint8(bands: ndarray, dtype: str | None) -> ndarray[uint8]: """Convert Banded data with NaNs (missing) into a uint8 data cube. - 99.99% of the time this will simply pass through the data coercing it back - into unsigned ints and setting the missing values to 0 that will be masked - as transparent in the output png. + Nearly all of the time this will simply pass through the data coercing it + back into unsigned ints and setting the missing values to 0 that will be + masked as transparent in the output png. There is a some small non-zero chance that the input RGB image was 16-bit and if any of the values exceed 255, we must normalize all of input data to From 41bad902d5fbf752715faa93e637e2530c1a07a3 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 17 Dec 2024 16:19:43 -0700 Subject: [PATCH 15/18] DAS-2276: clarify comment and rename variable. --- hybig/browse.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index b709955..c5237fb 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -360,13 +360,30 @@ def image_driver(mime: str) -> str: def prepare_raster_for_writing( raster: ndarray, driver: str, - input_bands: int, + band_count: int, ) -> tuple[ndarray, dict | None]: - """Remove alpha layer if writing a jpeg.""" + """Standardize raster data for writing to browse image. + + Args: + raster: Input raster data array + driver: Output image format ('JPEG' or 'PNG') + band_count: Number of bands in original input data + + The function handles two special cases: + - JPEG output with 4-band data -> Drop alpha channel and return 3-band RGB + - PNG output with single-band data -> Convert to paletted format + + Returns: + tuple: (prepared_raster, color_map) where: + - prepared_raster is the processed ndarray + - color_map is either None or a dict mapping palette indices to RGBA values + + + """ if driver == 'JPEG' and raster.shape[0] == 4: return raster[0:3, :, :], None - if driver == 'PNG' and input_bands == 1: + if driver == 'PNG' and band_count == 1: # we only paletize single band input data return palettize_raster(raster) From b6aadd039215696f80034b2b5774772e0a0b8cdf Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 17 Dec 2024 16:25:47 -0700 Subject: [PATCH 16/18] DAS-2276: prepare_raster_for_writing -> standardize_raster_for_writing --- hybig/browse.py | 4 ++-- tests/test_service/test_adapter.py | 4 ++-- tests/unit/test_browse.py | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index c5237fb..a9a020d 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -181,7 +181,7 @@ def create_browse_imagery( f'incorrect number of bands for image: {rio_in_array.rio.count}' ) - raster, color_map = prepare_raster_for_writing( + raster, color_map = standardize_raster_for_writing( raster, output_driver, rio_in_array.rio.count ) @@ -357,7 +357,7 @@ def image_driver(mime: str) -> str: return 'PNG' -def prepare_raster_for_writing( +def standardize_raster_for_writing( raster: ndarray, driver: str, band_count: int, diff --git a/tests/test_service/test_adapter.py b/tests/test_service/test_adapter.py index f4e87ff..eb11b65 100644 --- a/tests/test_service/test_adapter.py +++ b/tests/test_service/test_adapter.py @@ -19,7 +19,7 @@ from harmony_service.exceptions import HyBIGServiceError from hybig.browse import ( convert_mulitband_to_raster, - prepare_raster_for_writing, + standardize_raster_for_writing, ) from tests.utilities import Granule, create_stac @@ -460,7 +460,7 @@ def move_tif(*args, **kwargs): 'count': 3, } raster = convert_mulitband_to_raster(rio_data_array) - raster, color_map = prepare_raster_for_writing(raster, 'PNG', 3) + raster, color_map = standardize_raster_for_writing(raster, 'PNG', 3) dest = np.full( (expected_params['height'], expected_params['width']), diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index dc89feb..4f9c1c2 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -30,7 +30,7 @@ output_image_file, output_world_file, palettize_raster, - prepare_raster_for_writing, + standardize_raster_for_writing, validate_file_crs, validate_file_type, ) @@ -653,48 +653,48 @@ def test_convert_5_multiband_to_raster(self): 'Cannot create image from 5 band image. Expecting 3 or 4 bands.', ) - def test_prepare_raster_for_writing_jpeg_3band(self): + def test_standardize_raster_for_writing_jpeg_3band(self): raster = self.random.integers(255, size=(3, 5, 6)) count = 'irrelevant' driver = 'JPEG' expected_raster = np.copy(raster) expected_color_map = None - actual_raster, actual_color_map = prepare_raster_for_writing( + actual_raster, actual_color_map = standardize_raster_for_writing( raster, driver, count ) self.assertEqual(expected_color_map, actual_color_map) np.testing.assert_array_equal(expected_raster, actual_raster, strict=True) - def test_prepare_raster_for_writing_jpeg_4band(self): + def test_standardize_raster_for_writing_jpeg_4band(self): raster = self.random.integers(255, size=(4, 7, 8)) driver = 'JPEG' count = 'irrelevant' expected_raster = np.copy(raster[0:3, :, :]) expected_color_map = None - actual_raster, actual_color_map = prepare_raster_for_writing( + actual_raster, actual_color_map = standardize_raster_for_writing( raster, driver, count ) self.assertEqual(expected_color_map, actual_color_map) np.testing.assert_array_equal(expected_raster, actual_raster, strict=True) @patch('hybig.browse.palettize_raster') - def test_prepare_raster_for_writing_png_4band(self, palettize_mock): + def test_standardize_raster_for_writing_png_4band(self, palettize_mock): raster = self.random.integers(255, size=(4, 7, 8)) driver = 'PNG' count = 'not 1' - expected, _ = prepare_raster_for_writing(raster, driver, count) + expected, _ = standardize_raster_for_writing(raster, driver, count) np.testing.assert_array_equal(raster, expected, strict=True) palettize_mock.assert_not_called() @patch('hybig.browse.palettize_raster') - def test_prepare_raster_for_writing_png_3band(self, palettize_mock): + def test_standardize_raster_for_writing_png_3band(self, palettize_mock): raster = self.random.integers(255, size=(3, 7, 8)) driver = 'PNG' count = 'not 1' - expected, _ = prepare_raster_for_writing(raster, driver, count) + expected, _ = standardize_raster_for_writing(raster, driver, count) np.testing.assert_array_equal(raster, expected, strict=True) palettize_mock.assert_not_called() @@ -704,7 +704,7 @@ def test_prepare_1band_raster_for_writing_png(self, palettize_mock): driver = 'PNG' count = 1 palettize_mock.return_value = (None, None) - expected, _ = prepare_raster_for_writing(raster, driver, count) + expected, _ = standardize_raster_for_writing(raster, driver, count) palettize_mock.assert_called_with(raster) @patch('hybig.browse.Image') From bae52d6d9f73d29d9f486f7dcbcd19ddce2733aa Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 17 Dec 2024 16:33:50 -0700 Subject: [PATCH 17/18] DAS-2276: Clarifying comment --- hybig/browse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hybig/browse.py b/hybig/browse.py index a9a020d..83343ba 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -384,7 +384,8 @@ def standardize_raster_for_writing( return raster[0:3, :, :], None if driver == 'PNG' and band_count == 1: - # we only paletize single band input data + # Only paletize single band input data that has been converted to an + # RGBA raster. return palettize_raster(raster) return raster, None From 6ad622f0a44d911b15ea55ba90553e3a611ea2db Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 18 Dec 2024 14:27:17 -0700 Subject: [PATCH 18/18] DAS-2276: Fix typos --- hybig/browse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hybig/browse.py b/hybig/browse.py index 83343ba..8c120fd 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -275,7 +275,7 @@ def convert_to_uint8(bands: ndarray, dtype: str | None) -> ndarray[uint8]: def original_dtype(data_array: DataArray) -> str | None: """Return the original input data's type. - rastero_optn retains the input dtype in the encoding dictionary and is used + rastero_open retains the input dtype in the encoding dictionary and is used to understand what kind of casts are safe. """ @@ -384,7 +384,7 @@ def standardize_raster_for_writing( return raster[0:3, :, :], None if driver == 'PNG' and band_count == 1: - # Only paletize single band input data that has been converted to an + # Only palettize single band input data that has been converted to an # RGBA raster. return palettize_raster(raster)