ENH: Out-of-core architecture rewrite and filter optimizations

CMakeLists.txt

@@ -53,6 +53,14 @@ option(SIMPLNX_DOWNLOAD_TEST_FILES "Download the test files" ON)
 # ------------------------------------------------------------------------------
 option(SIMPLNX_WRITE_TEST_OUTPUT "Write unit test output files" OFF)
 
+# ------------------------------------------------------------------------------
+# Controls which algorithm paths are exercised by dual-dispatch unit tests.
+#   0 (Both)       - tests run with forceOoc=false AND forceOoc=true (default)
+#   1 (OocOnly)    - tests run with forceOoc=true only (use for OOC builds)
+#   2 (InCoreOnly) - tests run with forceOoc=false only (quick validation)
+# ------------------------------------------------------------------------------
+set(SIMPLNX_TEST_ALGORITHM_PATH "0" CACHE STRING "Algorithm paths to test: 0=Both, 1=OocOnly, 2=InCoreOnly")
+
 # ------------------------------------------------------------------------------
 # Is the SimplnxCore Plugin enabled [DEFAULT=ON]
 # ------------------------------------------------------------------------------
@@ -245,6 +253,9 @@ if(MSVC)
   target_compile_options(simplnx
     PRIVATE
       /MP
+      # Dream3dIO.cpp exceeds the default COMDAT section limit (C1128) in Debug
+      # after the out-of-core resolver refactor; /bigobj raises the limit.
+      /bigobj
   )
 endif()
 
@@ -261,6 +272,7 @@ if(SIMPLNX_ENABLE_MULTICORE)
   target_link_libraries(simplnx PUBLIC TBB::tbb)
 endif()
 
+
 target_link_libraries(simplnx
   PUBLIC
     fmt::fmt
@@ -293,6 +305,11 @@ if(SIMPLNX_ENABLE_LINK_FILESYSTEM)
 endif()
 
 set(SIMPLNX_GENERATED_DIR ${simplnx_BINARY_DIR}/generated)
+
+configure_file(
+  ${simplnx_SOURCE_DIR}/cmake/SimplnxConfig.hpp.in
+  ${SIMPLNX_GENERATED_DIR}/simplnx/Common/SimplnxConfig.hpp
+  @ONLY)
 set(SIMPLNX_GENERATED_HEADER_DIR ${simplnx_BINARY_DIR}/generated/simplnx)
 set(SIMPLNX_EXPORT_HEADER ${SIMPLNX_GENERATED_HEADER_DIR}/simplnx_export.hpp)
 
@@ -351,6 +368,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Common/Constants.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataTypeUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataVector.hpp
+  ${SIMPLNX_SOURCE_DIR}/Common/Extent.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/EulerAngle.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/Filesystem.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/IteratorUtility.hpp
@@ -380,6 +398,8 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/DataIOCollection.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataFactory.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataIOManager.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataStoreFormatResolver.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/InMemoryFormatResolver.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IOConstants.hpp
 
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/HDF5/DataIOManager.hpp
@@ -465,6 +485,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/DataStructure/DynamicListArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyDataStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyListStore.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyStringStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataStore.hpp
@@ -540,12 +561,14 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Plugin/PluginLoader.hpp
 
   ${SIMPLNX_SOURCE_DIR}/Utilities/ArrayCreationUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/SliceBufferedTransfer.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/AlignSections.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ArrayThreshold.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataArrayUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataGroupUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataObjectUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/AlgorithmDispatch.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FilePathGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ColorTableUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FileUtilities.hpp
@@ -556,6 +579,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringInterpretationUtilities.hpp
@@ -568,6 +592,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/SamplingUtils.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/SegmentFeatures.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TimeUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/UnionFind.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipRowItem.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/OStreamUtilities.hpp
@@ -639,6 +664,7 @@ set(SIMPLNX_SRCS
 
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/DataIOCollection.cpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataIOManager.cpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataStoreFormatResolver.cpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/CoreDataIOManager.cpp
 
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/HDF5/DataIOManager.cpp
@@ -770,6 +796,7 @@ set(SIMPLNX_SRCS
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.cpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ParallelDataAlgorithm.cpp

cmake/Plugin.cmake

@@ -383,6 +383,7 @@ function(create_simplnx_plugin_unit_test)
   target_compile_definitions(${UNIT_TEST_TARGET}
     PRIVATE
       SIMPLNX_BUILD_DIR="$<TARGET_FILE_DIR:simplnx_test>"
+      SIMPLNX_TEST_ALGORITHM_PATH=${SIMPLNX_TEST_ALGORITHM_PATH}
   )
 
   target_compile_options(${UNIT_TEST_TARGET}

cmake/SimplnxConfig.hpp.in

@@ -0,0 +1,12 @@
+#pragma once
+
+// This header is generated from cmake/SimplnxConfig.hpp.in by configure_file().
+// It carries compile-time configuration into every translation unit that links
+// simplnx, including MOC, via simplnx's PUBLIC generated include directory.
+// Delivering values through a header (not per-target compile definitions) keeps
+// them identical across all consumers, which is required for ODR safety when a
+// macro gates inline code in public headers.
+//
+// Out-of-core support is no longer a compile-time switch in core simplnx; OOC is
+// provided at runtime by a registered IO manager (see DataIOCollection). This
+// header is retained for future compile-time configuration values.

src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Common/ITKArrayHelper.hpp

@@ -840,7 +840,7 @@ Result<OutputActions> DataCheck(const DataStructure& dataStructure, const DataPa
   const auto& inputArray = dataStructure.getDataRefAs<IDataArray>(inputArrayPath);
   const auto& inputDataStore = inputArray.getIDataStoreRef();
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult<OutputActions>(Constants::k_OutOfCoreDataNotSupported,
                                           fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
@@ -862,7 +862,7 @@ Result<detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>> Execute(DataStr
 
   using ResultT = detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>;
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult(Constants::k_OutOfCoreDataNotSupported, fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
   }

src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp

@@ -30,7 +30,7 @@ std::string ComputeMD5HashTyped(const IDataArray& outputDataArray)
   usize arraySize = dataArray.getSize();
 
   MD5 md5;
-  if(outputDataArray.getDataFormat().empty())
+  if(outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore)
   {
     const T* dataPtr = dataArray.template getIDataStoreRefAs<DataStore<T>>().data();
     md5.update(reinterpret_cast<const uint8*>(dataPtr), arraySize * sizeof(T));
@@ -135,47 +135,7 @@ namespace ITKTestBase
 bool IsArrayInMemory(DataStructure& dataStructure, const DataPath& outputDataPath)
 {
   const auto& outputDataArray = dataStructure.getDataRefAs<IDataArray>(outputDataPath);
-  DataType outputDataType = outputDataArray.getDataType();
-
-  switch(outputDataType)
-  {
-  case DataType::float32: {
-    return dynamic_cast<const DataArray<float32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::float64: {
-    return dynamic_cast<const DataArray<float64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int8: {
-    return dynamic_cast<const DataArray<int8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint8: {
-    return dynamic_cast<const DataArray<uint8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int16: {
-    return dynamic_cast<const DataArray<int16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint16: {
-    return dynamic_cast<const DataArray<uint16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int32: {
-    return dynamic_cast<const DataArray<int32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint32: {
-    return dynamic_cast<const DataArray<uint32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int64: {
-    return dynamic_cast<const DataArray<int64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint64: {
-    return dynamic_cast<const DataArray<uint64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::boolean: {
-    [[fallthrough]];
-  }
-  default: {
-    return {};
-  }
-  }
+  return outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore;
 }
 //------------------------------------------------------------------------------
 std::string ComputeMd5Hash(DataStructure& dataStructure, const DataPath& outputDataPath)

src/Plugins/OrientationAnalysis/CMakeLists.txt

@@ -172,6 +172,8 @@ set(filter_algorithms
   AlignSectionsMisorientation
   AlignSectionsMutualInformation
   BadDataNeighborOrientationCheck
+  BadDataNeighborOrientationCheckScanline
+  BadDataNeighborOrientationCheckWorklist
   CAxisSegmentFeatures
   ComputeAvgCAxes
   ComputeAvgOrientations
@@ -186,9 +188,12 @@ set(filter_algorithms
   ComputeFZQuaternions
   ComputeGBCD
   ComputeGBCDMetricBased
-  ComputeGBCDPoleFigure
+  ComputeGBCDPoleFigureDirect
+  ComputeGBCDPoleFigureScanline
   ComputeGBPDMetricBased
   ComputeIPFColors
+  ComputeIPFColorsDirect
+  ComputeIPFColorsScanline
   ComputeKernelAvgMisorientations
   ComputeMisorientations
   ComputeQuaternionConjugate

src/Plugins/OrientationAnalysis/docs/AlignSectionsMisorientationFilter.md

@@ -45,6 +45,20 @@ In this new structure, what follows is what the created structures represent:
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
 
+## Algorithm
+
+### In-Core Path
+
+For each pair of adjacent Z-sections, the algorithm computes the misorientation between voxels across the section boundary. It tests candidate X-Y shifts to find the shift that minimizes the total misorientation between the two sections. All voxel comparisons use direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads pairs of adjacent Z-slices into local memory buffers using `copyIntoBuffer()`. All misorientation comparisons for a given slice pair operate entirely on the in-memory buffers. This converts what would otherwise be random cross-slice element access into sequential bulk reads, avoiding chunk thrashing when data is stored on disk in compressed chunks.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. By reading entire Z-slices in bulk rather than accessing individual voxels across slice boundaries, the algorithm avoids repeatedly decompressing the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines

src/Plugins/OrientationAnalysis/docs/AlignSectionsMutualInformationFilter.md

@@ -49,6 +49,20 @@ In this new structure, what follows is what the created structures represent:
 
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
+## Algorithm
+
+### In-Core Path
+
+Aligns Z-sections by maximizing the mutual information of orientation data between adjacent slices. The algorithm segments each slice into temporary features, bins the orientations, and computes joint and marginal histograms to evaluate mutual information at each candidate shift position. All orientation and feature ID data is accessed through direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads the orientation and phase data for each pair of adjacent Z-slices into local memory buffers using `copyIntoBuffer()` before computing histograms. This eliminates per-voxel out-of-core reads during the histogram binning and mutual information calculation, replacing them with two sequential bulk reads per slice pair.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. Histogram computation requires visiting every voxel in both slices multiple times (once per candidate shift), so eliminating per-element OOC access prevents repeated decompression of the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## References

src/Plugins/OrientationAnalysis/docs/BadDataNeighborOrientationCheckFilter.md

@@ -41,6 +41,32 @@ since there are no neighbors is the +-Z directions.
 
 Only the *Mask* value defining the cell as *good* or *bad* is changed. No other cell level array is modified.
 
+## Algorithm
+
+The algorithm operates in a multi-level iterative scheme, starting at level 6 (all 6 face-neighbors must agree) and decrementing to the user-specified *Required Number of Neighbors*. At each level, bad voxels are flipped to good if they have at least that many good face-neighbors with matching crystallographic orientation (misorientation below the tolerance). Starting strict and relaxing ensures high-confidence flips happen first, which can cascade to enable additional flips.
+
+### In-Core Path (BadDataNeighborOrientationCheckWorklist)
+
+When all arrays reside in contiguous in-memory storage, the algorithm uses a two-phase worklist approach:
+
+1. **Phase 1 (Initial count)**: A single linear scan counts matching good face-neighbors for every bad voxel, storing the count in a per-voxel array.
+2. **Phase 2 (Worklist propagation)**: For each level, a deque is seeded with all bad voxels meeting the threshold. As each voxel is flipped, its still-bad neighbors' counts are incremented. If a neighbor's count now meets the threshold, it is enqueued. This breadth-first flood-fill processes each voxel at most once per level, achieving O(flipped) amortized cost.
+
+### Out-of-Core Path (BadDataNeighborOrientationCheckScanline)
+
+When any of the quaternion, mask, or phase arrays are backed by chunked (OOC) disk storage, the algorithm uses a 3-slice rolling window over the Z axis:
+
+1. Three Z-slices of quaternions, phases, and mask data are maintained in memory (previous, current, next).
+2. For each bad voxel in the current slice, the count of matching good face-neighbors is recomputed on-the-fly using the rolling window buffers.
+3. If a voxel is flipped, the mask change is written back to the OOC store per-slice via `copyFromBuffer()`.
+4. The window shifts forward one Z-slice at a time, with only one new slice loaded per step.
+
+This approach trades recomputation (no persistent neighbor-count array) for strictly sequential I/O that avoids the random-access chunk thrashing that would occur with the worklist variant's BFS pattern.
+
+### Performance
+
+The in-core worklist variant is significantly faster for datasets that fit in RAM because each voxel is processed at most once per level (O(flipped) cost vs. O(N * passes) for the scanline variant). The OOC scanline variant is slower in absolute terms but avoids catastrophic performance degradation on disk-backed datasets where the worklist's random access pattern would trigger continuous chunk load/evict cycles. Memory usage is O(N) for the worklist variant vs. O(3 * sliceSize) for the scanline variant.
+
 ## Example Data
 
 | Example Input Image                                                       | Example Output Image                                                                                                                          |

src/Plugins/OrientationAnalysis/docs/CAxisSegmentFeaturesFilter.md

@@ -47,6 +47,22 @@ is to still use the 6 face neighbors ("Face Only") in order to stay consistent w
 |:--:|:--:|
 | ![Shared Edges & Points With Disconnected Region - "Face Only"](Images/SegmentFeatures/combination_face_only.png) | ![Shared Edges & Points With Disconnected Region - "All Connected"](Images/SegmentFeatures/combination_all_connected.png) |
 
+## Algorithm
+
+This filter segments voxels into features based on c-axis alignment, targeting hexagonal crystal systems. Voxels whose c-axes (the <001> crystallographic direction) are aligned within a user-defined tolerance are grouped into the same feature.
+
+### In-Core Path
+
+Uses a BFS-style flood fill where seed voxels are compared to their neighbors by computing the c-axis misalignment via direct array access with `operator[]`. Neighbors within tolerance are added to the current feature and queued for further expansion.
+
+### Out-of-Core Path
+
+Adapted to use sequential data access through the `SegmentFeatures` base class OOC support. The base class manages bulk I/O so that the flood-fill algorithm can proceed without triggering per-voxel OOC reads across chunk boundaries.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. Flood-fill naturally exhibits random access patterns as it grows regions across Z-slices, which can cause severe chunk thrashing with compressed on-disk storage. The OOC path mitigates this by leveraging the base class sequential access strategy. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines

src/Plugins/OrientationAnalysis/docs/ComputeAvgCAxesFilter.md

@@ -24,6 +24,22 @@ This filter will error out if **ALL** phases are non-hexagonal. Any non-hexagona
 
 The output is a direction vector for each feature.
 
+## Algorithm
+
+For each element, the quaternion orientation is converted to an active rotation matrix (by transposing the passive orientation matrix), which is then applied to the crystallographic c-axis direction <001> to find the c-axis location in the sample reference frame. These rotated c-axis vectors are accumulated per feature using a running average that checks sign consistency (flipping vectors whose dot product with the running average is negative). After all elements are processed, each feature's accumulated c-axis is normalized to produce the final average direction.
+
+### In-Core Path
+
+All cell-level arrays (feature IDs, phases, quaternions) and the feature-level output array are accessed through the AbstractDataStore API. Crystal structures are validated to ensure at least one hexagonal phase is present.
+
+### Out-of-Core Path
+
+Cell-level data is read in sequential 4096-tuple chunks via `copyIntoBuffer`, with separate buffers for feature IDs, cell phases, and quaternions. Feature-level accumulation is performed in a local buffer (`avgCAxesCache`) that is bulk-read at the start and bulk-written back via `copyFromBuffer` after the normalization pass. The ensemble-level crystal structures array is cached locally at startup to avoid per-element OOC overhead.
+
+### Performance
+
+The chunked sequential reads convert what would be millions of individual virtual dispatch calls into a small number of bulk I/O operations. Since the feature-level buffer is proportional to the number of grains (thousands) rather than voxels (millions), it fits comfortably in memory even when cell data is on disk.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines