From bcf30740c2c280e65e26ae94621a7493d3c30b5a Mon Sep 17 00:00:00 2001 From: Jeff Hammond Date: Thu, 16 Jul 2026 05:28:16 +0000 Subject: [PATCH] Add optional single-thread cutoff for small level-3 problems Below a work threshold (m*n*k), thread-spawn and barrier overhead dominates tiny GEMMs -- e.g. 16x16x16 runs ~10x slower with many threads than with one. Add a size-gated cutoff to the native and SUP level-3 thread decorators that forces a single thread when m*n*k < BLIS_SMALL_MT_THRESHOLD. The threshold is precision-aware: single-precision real GEMM uses a larger register tile (8x12 vs 6x8) so its small matrices have too few micro-tiles to parallelize, and its crossover is ~8x higher than double's. Opt-in: define BLIS_SMALL_MT_THRESHOLD (e.g. 200000) to enable; undefined leaves behavior unchanged. Threading policy only -- results are identical. Co-Authored-By: Claude Opus 4.8 (1M context) --- frame/3/bli_l3_decor.c | 22 ++++++++++++++++++++++ frame/3/bli_l3_sup_decor.c | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/frame/3/bli_l3_decor.c b/frame/3/bli_l3_decor.c index 6cbf968440..4d597c72f7 100644 --- a/frame/3/bli_l3_decor.c +++ b/frame/3/bli_l3_decor.c @@ -103,6 +103,28 @@ void bli_l3_thread_decorator if ( rntm != NULL ) rntm_l = *rntm; else bli_rntm_init_from_global( &rntm_l ); +#ifdef BLIS_SMALL_MT_THRESHOLD + // Force single-threaded execution for problems too small for the cost of + // spawning threads and their barriers to pay off. Without this, tiny GEMMs + // spend nearly all their time in thread setup (e.g. on GB10, 16x16x16 ran + // at ~1 GFLOP/s with 10 threads vs ~11 single-threaded). Gated per-config. + { + const uint64_t work = ( uint64_t )bli_obj_length( c ) * + ( uint64_t )bli_obj_width ( c ) * + ( uint64_t )bli_obj_width ( a ); + // Single-precision real GEMM uses a larger register tile (8x12 vs 6x8), + // so small matrices have too few micro-tiles to parallelize well; its + // crossover to worthwhile threading is ~8x larger than double's. + uint64_t thresh = ( uint64_t )( BLIS_SMALL_MT_THRESHOLD ); + if ( bli_obj_is_float( c ) ) thresh *= 8; + if ( work < thresh ) + { + bli_rntm_set_ways_only( 1, 1, 1, 1, 1, &rntm_l ); + bli_rntm_set_num_threads_only( 1, &rntm_l ); + } + } +#endif + // Set the number of ways for each loop, if needed, depending on what // kind of information is already stored in the rntm_t object. bli_rntm_factorize diff --git a/frame/3/bli_l3_sup_decor.c b/frame/3/bli_l3_sup_decor.c index 1f2c59dea0..b7de1bd09a 100644 --- a/frame/3/bli_l3_sup_decor.c +++ b/frame/3/bli_l3_sup_decor.c @@ -109,6 +109,24 @@ err_t bli_l3_sup_thread_decorator { rntm_t rntm_l = *rntm; +#ifdef BLIS_SMALL_MT_THRESHOLD + // Force single-threaded execution for problems too small for thread-spawn + // overhead to pay off. Small matrices take the SUP path, so this check is + // the one that actually fires for tiny GEMMs. Gated per-config. + { + const uint64_t work = ( uint64_t )bli_obj_length( c ) * + ( uint64_t )bli_obj_width ( c ) * + ( uint64_t )bli_obj_width ( a ); + uint64_t thresh = ( uint64_t )( BLIS_SMALL_MT_THRESHOLD ); + if ( bli_obj_is_float( c ) ) thresh *= 8; + if ( work < thresh ) + { + bli_rntm_set_ways_only( 1, 1, 1, 1, 1, &rntm_l ); + bli_rntm_set_num_threads_only( 1, &rntm_l ); + } + } +#endif + // Query the threading implementation and the number of threads requested. timpl_t ti = bli_rntm_thread_impl( &rntm_l ); dim_t nt = bli_rntm_num_threads( &rntm_l );