From e9115207d9a52d9b8ea34bc8f1116dcac20f6bdd Mon Sep 17 00:00:00 2001 From: Chris Twigg Date: Mon, 10 Nov 2025 15:25:49 -0800 Subject: [PATCH 1/2] Support rendering text in the rasterizer. (#786) Summary: I find myself wanting to add text labels to things in the image, currently the only way to do that is to fork out to OpenCV. It would be neat if we could write text to the image but support z-buffer checking, and if we didn't have to fork out to a separate library. Reviewed By: jeongseok-meta, cstollmeta Differential Revision: D85690037 --- cmake/build_variables.bzl | 3 + momentum/rasterizer/text_rasterizer.cpp | 500 ++++++++++++++++++ momentum/rasterizer/text_rasterizer.h | 89 ++++ .../test/rasterizer/test_text_rasterizer.cpp | 170 ++++++ pymomentum/renderer/renderer_pybind.cpp | 74 +++ pymomentum/renderer/software_rasterizer.cpp | 156 ++++++ pymomentum/renderer/software_rasterizer.h | 27 + 7 files changed, 1019 insertions(+) create mode 100644 momentum/rasterizer/text_rasterizer.cpp create mode 100644 momentum/rasterizer/text_rasterizer.h create mode 100644 momentum/test/rasterizer/test_text_rasterizer.cpp diff --git a/cmake/build_variables.bzl b/cmake/build_variables.bzl index 5c5f0a8904..94085cbab0 100644 --- a/cmake/build_variables.bzl +++ b/cmake/build_variables.bzl @@ -700,6 +700,7 @@ rasterizer_public_headers = [ "rasterizer/image.h", "rasterizer/rasterizer.h", "rasterizer/tensor.h", + "rasterizer/text_rasterizer.h", "rasterizer/utility.h", ] @@ -708,12 +709,14 @@ rasterizer_sources = [ "rasterizer/geometry.cpp", "rasterizer/rasterizer.cpp", "rasterizer/image.cpp", + "rasterizer/text_rasterizer.cpp", ] rasterizer_test_sources = [ "test/rasterizer/test_camera.cpp", "test/rasterizer/test_geometry.cpp", "test/rasterizer/test_software_rasterizer.cpp", + "test/rasterizer/test_text_rasterizer.cpp", ] #=========== diff --git a/momentum/rasterizer/text_rasterizer.cpp b/momentum/rasterizer/text_rasterizer.cpp new file mode 100644 index 0000000000..d2e387a7dd --- /dev/null +++ b/momentum/rasterizer/text_rasterizer.cpp @@ -0,0 +1,500 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "momentum/rasterizer/text_rasterizer.h" + +#include +#include + +namespace momentum::rasterizer { + +namespace { + +constexpr uint32_t kTextureWidth = 144; +constexpr uint32_t kTextureHeight = 240; +constexpr uint32_t kNumCharsWidth = 16; +constexpr uint32_t kNumCharsHeight = 16; +constexpr uint32_t kPadding = 1; +constexpr uint32_t kCharWidthInImage = kTextureWidth / kNumCharsWidth; +constexpr uint32_t kCharHeightInImage = kTextureHeight / kNumCharsHeight; +constexpr uint32_t kCharWidth = kCharWidthInImage - 2 * kPadding; +constexpr uint32_t kCharHeight = kCharHeightInImage - 2 * kPadding; + +// Proggy clean font from https://github.com/bluescan/proggyfonts +// License is MIT license, see https://github.com/bluescan/proggyfonts/blob/master/LICENSE +const struct { + uint32_t width; + uint32_t height; + std::array pixelData; +} fontPixmap = { + 144, + 240, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 6, 2, 0, 8, 0, 32, 0, 0, 0, 0, 0, 0, 0, 16, + 0, 192, 64, 129, 5, 2, 0, 8, 48, 32, 0, 0, 0, 0, 0, 0, 0, 16, 128, + 64, 64, 129, 5, 2, 0, 8, 16, 32, 0, 0, 0, 0, 0, 0, 0, 16, 224, 64, + 0, 128, 5, 2, 0, 8, 16, 32, 0, 0, 0, 0, 0, 0, 0, 252, 240, 64, 0, + 128, 133, 31, 63, 14, 16, 224, 241, 193, 3, 0, 0, 0, 0, 16, 240, 64, 0, 0, + 5, 0, 4, 8, 16, 32, 0, 65, 0, 0, 0, 0, 0, 16, 192, 64, 0, 0, 5, + 0, 4, 8, 16, 32, 128, 0, 0, 0, 0, 0, 0, 16, 128, 64, 64, 1, 5, 0, + 4, 8, 16, 32, 0, 0, 0, 0, 0, 0, 0, 16, 0, 192, 0, 0, 5, 0, 4, + 8, 16, 32, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4, 8, + 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 8, 32, 16, 0, 0, + 0, 0, 0, 0, 32, 0, 32, 160, 128, 2, 129, 8, 6, 8, 16, 32, 0, 0, 0, + 0, 0, 0, 32, 0, 32, 160, 128, 130, 71, 5, 9, 8, 16, 32, 0, 0, 0, 0, + 0, 0, 16, 0, 32, 0, 224, 71, 65, 5, 9, 0, 8, 64, 64, 128, 0, 0, 0, + 0, 16, 0, 32, 0, 64, 65, 129, 2, 38, 0, 8, 64, 80, 129, 0, 0, 0, 0, + 8, 0, 32, 0, 64, 129, 3, 10, 41, 0, 8, 64, 224, 224, 3, 128, 15, 0, 8, + 0, 32, 0, 240, 3, 5, 21, 17, 0, 8, 64, 80, 129, 0, 0, 0, 0, 4, 0, + 0, 0, 160, 0, 5, 21, 17, 0, 8, 64, 64, 128, 128, 0, 0, 2, 4, 0, 32, + 0, 160, 192, 131, 8, 46, 0, 16, 32, 0, 0, 128, 0, 0, 2, 2, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 16, 32, 0, 0, 128, 0, 0, 0, 2, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32, 16, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 32, 224, 192, 1, 132, 15, 12, 62, 56, + 112, 0, 0, 0, 0, 0, 0, 28, 68, 48, 16, 33, 2, 134, 0, 2, 32, 68, 136, + 0, 0, 0, 0, 0, 0, 34, 68, 40, 0, 1, 2, 133, 0, 1, 16, 68, 136, 64, + 64, 0, 6, 0, 3, 32, 84, 32, 128, 128, 129, 132, 7, 15, 16, 56, 136, 64, 64, + 128, 129, 31, 12, 16, 84, 32, 64, 0, 66, 4, 8, 17, 8, 68, 240, 0, 0, 96, + 0, 0, 48, 8, 68, 32, 32, 0, 194, 15, 8, 17, 8, 68, 128, 0, 0, 128, 129, + 31, 12, 8, 68, 32, 16, 32, 2, 132, 8, 17, 4, 68, 64, 64, 64, 0, 6, 0, + 3, 0, 56, 248, 240, 193, 1, 4, 7, 14, 4, 56, 48, 64, 64, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 96, 240, 128, 195, 131, + 15, 31, 56, 132, 112, 224, 32, 68, 192, 24, 35, 24, 68, 96, 16, 65, 68, 132, 0, + 1, 68, 132, 32, 128, 32, 66, 192, 24, 35, 36, 178, 144, 16, 33, 64, 136, 0, 1, + 2, 132, 32, 128, 32, 65, 64, 21, 37, 66, 170, 144, 240, 33, 64, 136, 7, 15, 2, + 252, 32, 128, 160, 64, 64, 21, 37, 66, 170, 240, 16, 34, 64, 136, 0, 1, 114, 132, + 32, 128, 224, 64, 64, 18, 41, 66, 114, 8, 17, 34, 64, 136, 0, 1, 66, 132, 32, + 128, 32, 65, 64, 18, 41, 66, 4, 8, 17, 66, 68, 132, 0, 1, 68, 132, 32, 128, + 32, 66, 64, 16, 49, 36, 120, 8, 241, 129, 195, 131, 15, 1, 56, 132, 112, 112, 32, + 196, 71, 16, 49, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 65, 0, 7, 4, 0, 60, 96, + 240, 192, 227, 143, 144, 32, 65, 132, 4, 241, 67, 64, 0, 4, 4, 0, 68, 144, 16, + 33, 4, 129, 144, 32, 73, 132, 4, 1, 66, 128, 0, 4, 10, 0, 68, 8, 17, 33, + 0, 129, 16, 17, 73, 72, 136, 0, 65, 128, 0, 4, 10, 0, 68, 8, 17, 193, 0, + 129, 16, 17, 85, 48, 80, 128, 64, 0, 1, 4, 17, 0, 60, 8, 241, 0, 3, 129, + 16, 10, 85, 48, 32, 64, 64, 0, 1, 4, 17, 0, 4, 8, 145, 0, 4, 129, 16, + 10, 54, 72, 32, 32, 64, 0, 2, 4, 0, 0, 4, 144, 16, 33, 4, 129, 16, 4, + 34, 132, 32, 16, 64, 0, 2, 4, 0, 0, 4, 96, 17, 194, 3, 1, 15, 4, 34, + 132, 32, 240, 67, 0, 4, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 64, 0, 4, 4, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 192, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 8, 0, 16, 0, 0, 4, 0, 28, 0, 4, 32, 128, 32, 128, 1, 0, + 0, 0, 16, 0, 16, 0, 0, 4, 0, 2, 0, 4, 0, 0, 32, 0, 1, 0, 0, + 0, 0, 0, 16, 0, 0, 4, 0, 2, 0, 4, 0, 0, 32, 0, 1, 0, 0, 0, + 0, 112, 240, 192, 129, 7, 7, 15, 60, 60, 48, 192, 32, 2, 193, 13, 15, 28, 0, + 128, 16, 33, 66, 132, 8, 2, 34, 68, 32, 128, 32, 1, 65, 18, 17, 34, 0, 240, + 16, 33, 64, 132, 15, 2, 34, 68, 32, 128, 160, 0, 65, 18, 17, 34, 0, 136, 16, + 33, 64, 132, 0, 2, 34, 68, 32, 128, 224, 0, 65, 18, 17, 34, 0, 136, 16, 33, + 66, 132, 8, 2, 34, 68, 32, 128, 32, 1, 65, 18, 17, 34, 0, 240, 240, 192, 129, + 7, 7, 2, 60, 68, 32, 128, 32, 2, 65, 18, 17, 28, 0, 0, 0, 0, 0, 0, + 0, 0, 32, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3, 129, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, + 0, 1, 2, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, + 1, 2, 0, 0, 60, 240, 208, 192, 131, 135, 8, 17, 65, 68, 136, 240, 129, 0, 1, + 2, 0, 0, 68, 136, 48, 33, 128, 128, 8, 17, 73, 40, 136, 0, 129, 0, 1, 2, + 39, 0, 68, 136, 16, 192, 128, 128, 8, 10, 73, 16, 136, 128, 96, 0, 1, 140, 28, + 0, 68, 136, 16, 0, 129, 128, 8, 10, 85, 16, 136, 64, 128, 0, 1, 2, 0, 0, + 68, 136, 16, 0, 130, 128, 8, 4, 54, 40, 136, 32, 128, 0, 1, 2, 0, 0, 60, + 240, 16, 224, 1, 7, 15, 4, 34, 68, 240, 240, 129, 0, 1, 2, 0, 0, 4, 128, + 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 1, 2, 0, 0, 4, 128, 0, + 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 3, 129, 1, 0, 0, 4, 128, 0, 0, + 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, + 8, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 4, 8, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 31, 62, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 4, 62, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 224, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, + 0, 0, 0, 0, 0, 0, 0, 4, 28, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 32, 0, 128, 1, 64, 16, 4, 34, 0, 248, 96, 0, 0, 0, 0, 31, 0, 0, 0, + 64, 64, 32, 72, 16, 4, 2, 0, 4, 129, 0, 0, 0, 128, 32, 0, 0, 32, 224, + 64, 192, 135, 8, 4, 12, 0, 100, 225, 128, 2, 0, 128, 38, 0, 0, 32, 80, 225, + 65, 4, 5, 4, 20, 0, 20, 145, 64, 1, 0, 128, 42, 0, 0, 32, 80, 64, 64, + 4, 2, 0, 24, 0, 20, 225, 160, 192, 3, 128, 38, 0, 0, 32, 80, 64, 64, 132, + 15, 4, 32, 0, 100, 1, 64, 1, 2, 128, 42, 0, 0, 32, 80, 33, 192, 7, 2, + 4, 34, 0, 4, 1, 128, 2, 2, 128, 32, 0, 0, 32, 224, 224, 35, 8, 2, 4, + 28, 0, 248, 0, 0, 0, 2, 0, 31, 0, 0, 0, 64, 0, 0, 0, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 8, 16, 0, 24, 0, 112, 224, 0, 2, 0, 0, 0, 0, 16, 96, 0, 64, 132, + 136, 17, 0, 36, 0, 128, 0, 1, 1, 0, 62, 0, 0, 24, 144, 0, 64, 130, 4, + 10, 8, 36, 32, 64, 192, 0, 0, 0, 23, 0, 0, 16, 144, 0, 64, 130, 4, 11, + 0, 24, 32, 32, 0, 1, 128, 8, 23, 0, 0, 16, 144, 160, 64, 129, 2, 6, 8, + 0, 248, 240, 224, 0, 128, 8, 22, 28, 0, 56, 96, 64, 1, 1, 142, 5, 8, 0, + 32, 0, 0, 0, 128, 8, 20, 28, 0, 0, 0, 128, 130, 4, 17, 18, 4, 0, 32, + 0, 0, 0, 128, 8, 20, 28, 0, 0, 0, 64, 129, 6, 9, 26, 2, 0, 0, 0, + 0, 0, 128, 9, 20, 0, 0, 0, 0, 160, 64, 143, 4, 61, 34, 0, 248, 0, 0, + 0, 128, 22, 20, 0, 0, 0, 0, 0, 64, 132, 28, 17, 28, 0, 0, 0, 0, 0, + 128, 0, 20, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, + 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 64, 192, 128, 2, 0, 6, 0, 0, 8, + 64, 64, 0, 0, 1, 4, 4, 0, 32, 32, 32, 65, 129, 4, 9, 0, 0, 16, 32, + 160, 64, 1, 2, 2, 10, 20, 0, 0, 0, 0, 0, 0, 9, 60, 56, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 48, 96, 192, 128, 1, 3, 6, 10, 68, 124, 248, 240, 225, + 131, 3, 7, 14, 28, 72, 144, 32, 65, 130, 4, 9, 10, 2, 4, 8, 16, 32, 0, + 1, 2, 4, 8, 72, 144, 32, 65, 130, 4, 9, 25, 2, 4, 8, 16, 32, 0, 1, + 2, 4, 8, 120, 240, 224, 193, 131, 7, 15, 15, 2, 60, 120, 240, 224, 1, 1, 2, + 4, 8, 132, 8, 17, 34, 68, 136, 144, 8, 2, 4, 8, 16, 32, 0, 1, 2, 4, + 8, 132, 8, 17, 34, 68, 136, 144, 8, 68, 4, 8, 16, 32, 0, 1, 2, 4, 8, + 132, 8, 17, 34, 68, 136, 144, 56, 56, 124, 248, 240, 225, 131, 3, 7, 14, 28, 0, + 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 64, 0, 1, 3, + 10, 0, 0, 0, 32, 128, 128, 1, 0, 4, 0, 0, 0, 80, 128, 128, 128, 4, 5, + 18, 0, 0, 64, 64, 64, 130, 4, 2, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 176, 0, 0, 0, 0, 0, 0, 1, 34, 60, 24, 193, 128, 1, 3, 6, 12, 0, + 72, 8, 17, 34, 68, 72, 16, 1, 34, 68, 40, 33, 65, 130, 4, 9, 18, 34, 164, + 8, 17, 34, 68, 72, 16, 15, 30, 132, 40, 17, 34, 68, 136, 16, 33, 20, 164, 8, + 17, 34, 68, 136, 8, 17, 34, 158, 72, 17, 34, 68, 136, 16, 33, 8, 148, 8, 17, + 34, 68, 8, 5, 17, 66, 132, 72, 17, 34, 68, 136, 16, 33, 20, 148, 8, 17, 34, + 68, 8, 2, 15, 66, 68, 136, 33, 65, 130, 4, 9, 18, 34, 72, 8, 17, 34, 68, + 8, 2, 1, 66, 60, 136, 193, 128, 1, 3, 6, 12, 0, 52, 240, 224, 193, 131, 7, + 2, 1, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 64, 64, + 128, 2, 0, 5, 0, 0, 8, 32, 64, 0, 128, 0, 2, 4, 20, 32, 32, 160, 64, + 129, 2, 2, 0, 0, 16, 16, 160, 64, 1, 1, 1, 10, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 112, 224, 192, 129, 3, + 7, 27, 28, 56, 112, 224, 192, 129, 1, 3, 6, 12, 64, 128, 0, 1, 2, 4, 8, + 36, 34, 68, 136, 16, 33, 2, 1, 2, 4, 8, 120, 240, 224, 193, 131, 7, 15, 63, + 2, 124, 248, 240, 225, 3, 1, 2, 4, 8, 68, 136, 16, 33, 66, 132, 136, 4, 2, + 4, 8, 16, 32, 0, 1, 2, 4, 8, 68, 136, 16, 33, 66, 132, 136, 36, 34, 68, + 136, 16, 33, 2, 1, 2, 4, 8, 120, 240, 224, 193, 131, 7, 15, 31, 28, 56, 112, + 224, 192, 1, 1, 2, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 88, 80, 32, 0, 1, 1, 10, 0, 0, 0, 16, 64, 128, 0, 0, 4, 0, 0, + 32, 40, 64, 128, 128, 2, 5, 10, 0, 0, 32, 32, 64, 129, 2, 2, 1, 20, 80, + 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, 64, 120, + 224, 192, 129, 3, 7, 14, 8, 56, 136, 16, 33, 66, 132, 8, 15, 34, 120, 136, 16, + 33, 66, 132, 8, 17, 0, 100, 136, 16, 33, 66, 132, 8, 17, 34, 68, 136, 16, 33, + 66, 132, 8, 17, 62, 84, 136, 16, 33, 66, 132, 8, 17, 34, 68, 136, 16, 33, 66, + 132, 8, 17, 0, 84, 136, 16, 33, 66, 132, 8, 17, 34, 68, 136, 16, 33, 66, 132, + 8, 17, 8, 76, 136, 16, 33, 66, 132, 8, 17, 34, 56, 136, 224, 192, 129, 3, 7, + 14, 0, 56, 240, 224, 193, 131, 7, 15, 15, 60, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 0, 0, 0, 0, 0, 8, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 8, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 7, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0}}; + +bool getPixelFromBitmap(uint32_t x, uint32_t y) { + const size_t pixelIndex = y * kTextureWidth + x; + const size_t byteIndex = pixelIndex / 8; + const size_t bitIndex = pixelIndex - byteIndex * 8; + return ((fontPixmap.pixelData[byteIndex] & (static_cast(1) << bitIndex)) != 0); +} + +void renderCharacter( + char c, + int imageX, + int imageY, + int textScale, + float depth, + const Eigen::Vector3f& color, + Span2f zBuffer, + Span3f rgbBuffer) { + const uint32_t charCode = static_cast(c); + const uint32_t charRow = charCode / kNumCharsWidth; + const uint32_t charCol = charCode - (charRow * kNumCharsWidth); + + const uint32_t minTextureX = charCol * kCharWidthInImage + kPadding; + const uint32_t minTextureY = charRow * kCharHeightInImage + kPadding; + + const int imageHeight = static_cast(zBuffer.extent(0)); + const int imageWidth = static_cast(zBuffer.extent(1)); + + for (uint32_t charY = 0; charY < kCharHeight; ++charY) { + for (uint32_t charX = 0; charX < kCharWidth; ++charX) { + const uint32_t texX = minTextureX + charX; + const uint32_t texY = minTextureY + charY; + + if (getPixelFromBitmap(texX, texY)) { + for (int sy = 0; sy < textScale; ++sy) { + for (int sx = 0; sx < textScale; ++sx) { + const int pixelX = imageX + static_cast(charX) * textScale + sx; + const int pixelY = imageY + static_cast(charY) * textScale + sy; + + if (pixelX >= 0 && pixelX < imageWidth && pixelY >= 0 && pixelY < imageHeight) { + if (depth <= zBuffer(pixelY, pixelX)) { + zBuffer(pixelY, pixelX) = depth; + if (!rgbBuffer.empty()) { + rgbBuffer(pixelY, pixelX, 0) = color.x(); + rgbBuffer(pixelY, pixelX, 1) = color.y(); + rgbBuffer(pixelY, pixelX, 2) = color.z(); + } + } + } + } + } + } + } + } +} + +void renderCharacter2D( + char c, + int imageX, + int imageY, + int textScale, + const Eigen::Vector3f& color, + Span3f rgbBuffer, + Span2f zBuffer) { + const uint32_t charCode = static_cast(c); + const uint32_t charRow = charCode / kNumCharsWidth; + const uint32_t charCol = charCode - (charRow * kNumCharsWidth); + + const uint32_t minTextureX = charCol * kCharWidthInImage + kPadding; + const uint32_t minTextureY = charRow * kCharHeightInImage + kPadding; + + const int imageHeight = static_cast(rgbBuffer.extent(0)); + const int imageWidth = static_cast(rgbBuffer.extent(1)); + + for (uint32_t charY = 0; charY < kCharHeight; ++charY) { + for (uint32_t charX = 0; charX < kCharWidth; ++charX) { + const uint32_t texX = minTextureX + charX; + const uint32_t texY = minTextureY + charY; + + if (getPixelFromBitmap(texX, texY)) { + for (int sy = 0; sy < textScale; ++sy) { + for (int sx = 0; sx < textScale; ++sx) { + const int pixelX = imageX + static_cast(charX) * textScale + sx; + const int pixelY = imageY + static_cast(charY) * textScale + sy; + + if (pixelX >= 0 && pixelX < imageWidth && pixelY >= 0 && pixelY < imageHeight) { + rgbBuffer(pixelY, pixelX, 0) = color.x(); + rgbBuffer(pixelY, pixelX, 1) = color.y(); + rgbBuffer(pixelY, pixelX, 2) = color.z(); + + if (!zBuffer.empty()) { + zBuffer(pixelY, pixelX) = 0.0f; + } + } + } + } + } + } + } +} + +} // namespace + +void rasterizeText( + gsl::span positionsWorld, + gsl::span texts, + const Camera& camera, + const Eigen::Matrix4f& modelMatrix, + float nearClip, + const Eigen::Vector3f& color, + int textScale, + Span2f zBuffer, + Span3f rgbBuffer, + float depthOffset, + const Eigen::Vector2f& imageOffset, + HorizontalAlignment horizontalAlignment, + VerticalAlignment verticalAlignment) { + MT_THROW_IF(positionsWorld.size() != texts.size(), "Number of positions and texts must be equal"); + + const int scaledCharWidth = static_cast(kCharWidth) * textScale; + const int scaledCharHeight = static_cast(kCharHeight) * textScale; + + const Eigen::Affine3f worldFromEye = camera.worldFromEye(); + const Eigen::Affine3f eyeFromWorld = worldFromEye.inverse(); + const auto& intrinsicsModel = camera.intrinsicsModel(); + + for (size_t i = 0; i < positionsWorld.size(); ++i) { + const Eigen::Vector3f worldPos = (modelMatrix * positionsWorld[i].homogeneous()).head<3>(); + const Eigen::Vector3f eyePos = eyeFromWorld * worldPos; + + if (eyePos.z() <= nearClip) { + continue; + } + + auto [imagePos, valid] = intrinsicsModel->project(eyePos); + if (!valid) { + continue; + } + + imagePos.z() += depthOffset; + + const std::string& text = texts[i]; + const int textWidth = static_cast(text.length()) * scaledCharWidth; + + int offsetX = 0; + switch (horizontalAlignment) { + case HorizontalAlignment::Left: + offsetX = 0; + break; + case HorizontalAlignment::Center: + offsetX = -textWidth / 2; + break; + case HorizontalAlignment::Right: + offsetX = -textWidth; + break; + } + + int offsetY = 0; + switch (verticalAlignment) { + case VerticalAlignment::Top: + offsetY = 0; + break; + case VerticalAlignment::Center: + offsetY = -scaledCharHeight / 2; + break; + case VerticalAlignment::Bottom: + offsetY = -scaledCharHeight; + break; + } + + int currentX = static_cast(imagePos.x() + imageOffset.x()) + offsetX; + int currentY = static_cast(imagePos.y() + imageOffset.y()) + offsetY; + + for (char c : text) { + renderCharacter(c, currentX, currentY, textScale, imagePos.z(), color, zBuffer, rgbBuffer); + currentX += scaledCharWidth; + } + } +} + +void rasterizeText2D( + gsl::span positionsImage, + gsl::span texts, + const Eigen::Vector3f& color, + int textScale, + Span3f rgbBuffer, + Span2f zBuffer, + const Eigen::Vector2f& imageOffset, + HorizontalAlignment horizontalAlignment, + VerticalAlignment verticalAlignment) { + if (positionsImage.size() != texts.size()) { + return; + } + + const int scaledCharWidth = static_cast(kCharWidth) * textScale; + const int scaledCharHeight = static_cast(kCharHeight) * textScale; + + for (size_t i = 0; i < positionsImage.size(); ++i) { + const std::string& text = texts[i]; + const int textWidth = static_cast(text.length()) * scaledCharWidth; + + int offsetX = 0; + switch (horizontalAlignment) { + case HorizontalAlignment::Left: + offsetX = 0; + break; + case HorizontalAlignment::Center: + offsetX = -textWidth / 2; + break; + case HorizontalAlignment::Right: + offsetX = -textWidth; + break; + } + + int offsetY = 0; + switch (verticalAlignment) { + case VerticalAlignment::Top: + offsetY = 0; + break; + case VerticalAlignment::Center: + offsetY = -scaledCharHeight / 2; + break; + case VerticalAlignment::Bottom: + offsetY = -scaledCharHeight; + break; + } + + int currentX = static_cast(positionsImage[i].x() + imageOffset.x()) + offsetX; + int currentY = static_cast(positionsImage[i].y() + imageOffset.y()) + offsetY; + + for (char c : text) { + renderCharacter2D(c, currentX, currentY, textScale, color, rgbBuffer, zBuffer); + currentX += scaledCharWidth; + } + } +} + +} // namespace momentum::rasterizer diff --git a/momentum/rasterizer/text_rasterizer.h b/momentum/rasterizer/text_rasterizer.h new file mode 100644 index 0000000000..d00cf9a7b1 --- /dev/null +++ b/momentum/rasterizer/text_rasterizer.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace momentum::rasterizer { + +/// Horizontal alignment options for text rendering +enum class HorizontalAlignment { + Left, + Center, + Right, +}; + +/// Vertical alignment options for text rendering +enum class VerticalAlignment { + Top, + Center, + Bottom, +}; + +/// Rasterize text at 3D world positions +/// +/// Projects 3D positions to image space using the camera and renders text strings at those +/// locations. Uses an embedded bitmap font for rendering. +/// +/// @param positionsWorld 3D positions in world coordinates where text should be rendered +/// @param texts Text strings to render at each position +/// @param camera Camera to render from +/// @param modelMatrix Model transformation matrix +/// @param nearClip Near clipping distance +/// @param color RGB color for the text +/// @param textScale Integer scaling factor for text size (1 = 1 pixel per font pixel) +/// @param zBuffer Input/output depth buffer (SIMD-aligned) +/// @param rgbBuffer Optional input/output RGB color buffer +/// @param imageOffset Pixel offset for positioning +/// @param horizontalAlignment Horizontal text alignment relative to position +/// @param verticalAlignment Vertical text alignment relative to position +void rasterizeText( + gsl::span positionsWorld, + gsl::span texts, + const Camera& camera, + const Eigen::Matrix4f& modelMatrix, + float nearClip, + const Eigen::Vector3f& color, + int textScale, + Span2f zBuffer, + Span3f rgbBuffer = {}, + float depthOffset = 0, + const Eigen::Vector2f& imageOffset = {0, 0}, + HorizontalAlignment horizontalAlignment = HorizontalAlignment::Left, + VerticalAlignment verticalAlignment = VerticalAlignment::Top); + +/// Rasterize text directly in 2D image space +/// +/// Renders text at 2D image positions without camera projection or depth testing. +/// +/// @param positionsImage 2D positions in image coordinates where text should be rendered +/// @param texts Text strings to render at each position +/// @param color RGB color for the text +/// @param textScale Integer scaling factor for text size (1 = 1 pixel per font pixel) +/// @param rgbBuffer Input/output RGB color buffer +/// @param zBuffer Optional depth buffer (fills with zeros when provided) +/// @param imageOffset Pixel offset for positioning +/// @param horizontalAlignment Horizontal text alignment relative to position +/// @param verticalAlignment Vertical text alignment relative to position +void rasterizeText2D( + gsl::span positionsImage, + gsl::span texts, + const Eigen::Vector3f& color, + int textScale, + Span3f rgbBuffer, + Span2f zBuffer = {}, + const Eigen::Vector2f& imageOffset = {0, 0}, + HorizontalAlignment horizontalAlignment = HorizontalAlignment::Left, + VerticalAlignment verticalAlignment = VerticalAlignment::Top); + +} // namespace momentum::rasterizer diff --git a/momentum/test/rasterizer/test_text_rasterizer.cpp b/momentum/test/rasterizer/test_text_rasterizer.cpp new file mode 100644 index 0000000000..92cc910736 --- /dev/null +++ b/momentum/test/rasterizer/test_text_rasterizer.cpp @@ -0,0 +1,170 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +#include + +using namespace momentum::rasterizer; + +TEST(TextRasterizer, BasicText3D) { + const int width = 200; + const int height = 100; + + OpenCVDistortionParametersT distortionParams; + auto intrinsics = std::make_shared( + width, height, width / 2.0f, height / 2.0f, width / 2.0f, height / 2.0f, distortionParams); + + Camera camera(intrinsics); + + auto zBuffer = makeRasterizerZBuffer(camera); + auto rgbBuffer = makeRasterizerRGBBuffer(camera); + + std::vector positions = {Eigen::Vector3f(0.0f, 0.0f, 1.5f)}; + std::vector texts = {"Hello"}; + + rasterizeText( + positions, + texts, + camera, + Eigen::Matrix4f::Identity(), + 0.1f, + Eigen::Vector3f(1.0f, 0.0f, 0.0f), + 1, + zBuffer.view(), + rgbBuffer.view()); + + int pixelsSet = 0; + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + if (zBuffer(y, x) < FLT_MAX) { + pixelsSet++; + EXPECT_NEAR(rgbBuffer(y, x, 0), 1.0f, 1e-5f); + EXPECT_NEAR(rgbBuffer(y, x, 1), 0.0f, 1e-5f); + EXPECT_NEAR(rgbBuffer(y, x, 2), 0.0f, 1e-5f); + EXPECT_NEAR(zBuffer(y, x), 1.5f, 1e-5f); + } + } + } + + EXPECT_GT(pixelsSet, 0); +} + +TEST(TextRasterizer, BasicText2D) { + const int width = 200; + const int height = 100; + + OpenCVDistortionParametersT distortionParams; + auto intrinsics = std::make_shared( + width, height, width / 2.0f, height / 2.0f, width / 2.0f, height / 2.0f, distortionParams); + + Camera camera(intrinsics); + + auto zBuffer = makeRasterizerZBuffer(camera); + auto rgbBuffer = makeRasterizerRGBBuffer(camera); + + std::vector positions = {Eigen::Vector2f(10.0f, 10.0f)}; + std::vector texts = {"Test"}; + + rasterizeText2D( + positions, texts, Eigen::Vector3f(0.0f, 1.0f, 0.0f), 1, rgbBuffer.view(), zBuffer.view()); + + int pixelsSet = 0; + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + if (zBuffer(y, x) < FLT_MAX) { + pixelsSet++; + EXPECT_NEAR(rgbBuffer(y, x, 0), 0.0f, 1e-5f); + EXPECT_NEAR(rgbBuffer(y, x, 1), 1.0f, 1e-5f); + EXPECT_NEAR(rgbBuffer(y, x, 2), 0.0f, 1e-5f); + EXPECT_NEAR(zBuffer(y, x), 0.0f, 1e-5f); + } + } + } + + EXPECT_GT(pixelsSet, 0); +} + +TEST(TextRasterizer, TextScaling) { + const int width = 400; + const int height = 200; + + OpenCVDistortionParametersT distortionParams; + auto intrinsics = std::make_shared( + width, height, width / 2.0f, height / 2.0f, width / 2.0f, height / 2.0f, distortionParams); + + Camera camera(intrinsics); + + auto rgbBuffer1 = makeRasterizerRGBBuffer(camera); + auto rgbBuffer2 = makeRasterizerRGBBuffer(camera); + + std::vector positions = {Eigen::Vector2f(10.0f, 10.0f)}; + std::vector texts = {"A"}; + + rasterizeText2D(positions, texts, Eigen::Vector3f(1.0f, 1.0f, 1.0f), 1, rgbBuffer1.view()); + + int pixelsScale1 = 0; + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + if (rgbBuffer1(y, x, 0) > 0.5f) { + pixelsScale1++; + } + } + } + + rasterizeText2D(positions, texts, Eigen::Vector3f(1.0f, 1.0f, 1.0f), 2, rgbBuffer2.view()); + + int pixelsScale2 = 0; + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + if (rgbBuffer2(y, x, 0) > 0.5f) { + pixelsScale2++; + } + } + } + + EXPECT_GT(pixelsScale1, 0); + EXPECT_GT(pixelsScale2, pixelsScale1); + EXPECT_NEAR(static_cast(pixelsScale2) / pixelsScale1, 4.0f, 1.0f); +} + +TEST(TextRasterizer, MultipleTexts) { + const int width = 400; + const int height = 200; + + OpenCVDistortionParametersT distortionParams; + auto intrinsics = std::make_shared( + width, height, width / 2.0f, height / 2.0f, width / 2.0f, height / 2.0f, distortionParams); + + Camera camera(intrinsics); + + auto zBuffer = makeRasterizerZBuffer(camera); + auto rgbBuffer = makeRasterizerRGBBuffer(camera); + + std::vector positions = { + Eigen::Vector2f(10.0f, 10.0f), Eigen::Vector2f(10.0f, 30.0f)}; + std::vector texts = {"Line1", "Line2"}; + + rasterizeText2D( + positions, texts, Eigen::Vector3f(1.0f, 0.0f, 1.0f), 1, rgbBuffer.view(), zBuffer.view()); + + int pixelsSet = 0; + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + if (zBuffer(y, x) < FLT_MAX) { + pixelsSet++; + EXPECT_NEAR(rgbBuffer(y, x, 0), 1.0f, 1e-5f); + EXPECT_NEAR(rgbBuffer(y, x, 1), 0.0f, 1e-5f); + EXPECT_NEAR(rgbBuffer(y, x, 2), 1.0f, 1e-5f); + } + } + } + + EXPECT_GT(pixelsSet, 0); +} diff --git a/pymomentum/renderer/renderer_pybind.cpp b/pymomentum/renderer/renderer_pybind.cpp index 55151aa3a0..b6959da44c 100644 --- a/pymomentum/renderer/renderer_pybind.cpp +++ b/pymomentum/renderer/renderer_pybind.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -645,6 +646,18 @@ PYBIND11_MODULE(renderer, m) { .value("Ambient", momentum::rasterizer::LightType::Ambient) .value("Directional", momentum::rasterizer::LightType::Directional) .value("Point", momentum::rasterizer::LightType::Point); + + py::enum_( + m, "HorizontalAlignment", "Horizontal text alignment options.") + .value("Left", momentum::rasterizer::HorizontalAlignment::Left) + .value("Center", momentum::rasterizer::HorizontalAlignment::Center) + .value("Right", momentum::rasterizer::HorizontalAlignment::Right); + + py::enum_( + m, "VerticalAlignment", "Vertical text alignment options.") + .value("Top", momentum::rasterizer::VerticalAlignment::Top) + .value("Center", momentum::rasterizer::VerticalAlignment::Center) + .value("Bottom", momentum::rasterizer::VerticalAlignment::Bottom); py::class_( m, "Light", @@ -1459,4 +1472,65 @@ This is useful for rendering shadows using the classic projection shadows techni py::arg("light"), py::arg("plane_normal") = std::optional{}, py::arg("plane_origin") = std::optional{}); + + m.def( + "rasterize_text", + &rasterizeText, + R"(Rasterize text at 3D world positions. + +Projects 3D positions to image space using the camera and renders text strings at those locations using an embedded bitmap font. + +:param positions: (nTexts x 3) torch.Tensor of 3D positions in world coordinates. +:param texts: List of strings to render at each position. +:param camera: Camera to render from. +:param z_buffer: Z-buffer to render geometry onto; can be reused for multiple renders. +:param rgb_buffer: Optional RGB-buffer to render geometry onto. +:param color: RGB color for the text. Defaults to white (1, 1, 1). +:param text_scale: Integer scaling factor for text size (1 = 1 pixel per font pixel). Defaults to 1. +:param horizontal_alignment: Horizontal text alignment (Left, Center, or Right). Defaults to Left. +:param vertical_alignment: Vertical text alignment (Top, Center, or Bottom). Defaults to Top. +:param model_matrix: Additional matrix to apply to the model. Defaults to identity matrix. +:param near_clip: Clip any text closer than this depth. Defaults to 0.1. +:param depth_offset: Offset the depth values. Defaults to 0. +:param image_offset: Offset by (x, y) pixels in image space. +)", + py::arg("positions"), + py::arg("texts"), + py::arg("camera"), + py::arg("z_buffer"), + py::arg("rgb_buffer") = std::optional{}, + py::kw_only(), + py::arg("color") = Eigen::Vector3f(1.0f, 1.0f, 1.0f), + py::arg("text_scale") = 1, + py::arg("horizontal_alignment") = momentum::rasterizer::HorizontalAlignment::Left, + py::arg("vertical_alignment") = momentum::rasterizer::VerticalAlignment::Top, + py::arg("model_matrix") = std::optional{}, + py::arg("near_clip") = 0.1f, + py::arg("depth_offset") = 0.0f, + py::arg("image_offset") = std::optional{}); + + m.def( + "rasterize_text_2d", + &rasterizeText2D, + R"(Rasterize text directly in 2D image space without camera projection or depth testing. + +:param positions: (nTexts x 2) torch.Tensor of 2D positions in image coordinates. +:param texts: List of strings to render at each position. +:param rgb_buffer: RGB-buffer to render geometry onto. +:param color: RGB color for the text. Defaults to white (1, 1, 1). +:param text_scale: Integer scaling factor for text size (1 = 1 pixel per font pixel). Defaults to 1. +:param horizontal_alignment: Horizontal text alignment (Left, Center, or Right). Defaults to Left. +:param vertical_alignment: Vertical text alignment (Top, Center, or Bottom). Defaults to Top. +:param z_buffer: Optional Z-buffer to write zeros to for alpha matting. +:param image_offset: Offset by (x, y) pixels in image space. +)", + py::arg("positions"), + py::arg("texts"), + py::arg("rgb_buffer"), + py::arg("color") = Eigen::Vector3f(1.0f, 1.0f, 1.0f), + py::arg("text_scale") = 1, + py::arg("horizontal_alignment") = momentum::rasterizer::HorizontalAlignment::Left, + py::arg("vertical_alignment") = momentum::rasterizer::VerticalAlignment::Top, + py::arg("z_buffer") = std::optional{}, + py::arg("image_offset") = std::optional{}); } diff --git a/pymomentum/renderer/software_rasterizer.cpp b/pymomentum/renderer/software_rasterizer.cpp index b45009b70b..2b6cb9c7f8 100644 --- a/pymomentum/renderer/software_rasterizer.cpp +++ b/pymomentum/renderer/software_rasterizer.cpp @@ -1495,6 +1495,162 @@ void rasterizeLines2D( } } +void rasterizeText( + at::Tensor positions, + const std::vector& texts, + const momentum::rasterizer::Camera& camera, + at::Tensor zBuffer, + std::optional rgbBuffer, + const std::optional& color, + int textScale, + momentum::rasterizer::HorizontalAlignment horizontalAlignment, + momentum::rasterizer::VerticalAlignment verticalAlignment, + const std::optional& modelMatrix, + float nearClip, + float depthOffset, + const std::optional& imageOffset) { + drjit::scoped_flush_denormals flushDenorm(true); + pybind11::gil_scoped_release release; + + const int nTextBindingId = -1; + const int heightBindingId = -2003; + const int widthBindingId = -2004; + + TensorChecker checker("rasterize_text"); + positions = checker.validateAndFixTensor( + positions, "positions", {nTextBindingId, 3}, {"nTexts", "xyz"}, at::kFloat, true, false); + + zBuffer = checker.validateAndFixTensor( + zBuffer, + "z_buffer", + {heightBindingId, widthBindingId}, + {"height", "width"}, + at::kFloat, + true, + false); + + if (rgbBuffer.has_value()) { + rgbBuffer = checker.validateAndFixTensor( + *rgbBuffer, + "rgb_buffer", + {heightBindingId, widthBindingId, 3}, + {"height", "width", "rgb"}, + at::kFloat, + true, + false); + } + + const int64_t numTexts = checker.getBoundValue(nTextBindingId); + if (numTexts != static_cast(texts.size())) { + throw std::runtime_error( + fmt::format( + "Mismatch between number of positions ({}) and texts ({})", numTexts, texts.size())); + } + + for (size_t iBatch = 0; iBatch < checker.getBatchSize(); ++iBatch) { + auto zBufferCur = zBuffer.select(0, iBatch); + auto positionsCur = positions.select(0, iBatch); + auto rgbBufferCur = + rgbBuffer.has_value() ? maybeSelect(rgbBuffer, iBatch) : std::optional{}; + + const Eigen::Ref positionsFlat = toEigenMap(positionsCur); + std::vector positionsVec; + positionsVec.reserve(numTexts); + for (int i = 0; i < numTexts; ++i) { + positionsVec.emplace_back(positionsFlat.segment<3>(3 * i)); + } + + momentum::rasterizer::rasterizeText( + positionsVec, + texts, + camera, + modelMatrix.value_or(Eigen::Matrix4f::Identity()), + nearClip, + color.value_or(Eigen::Vector3f::Ones()), + textScale, + make_mdspan(zBufferCur), + make_mdspan(rgbBufferCur), + depthOffset, + imageOffset.value_or(Eigen::Vector2f::Zero()), + horizontalAlignment, + verticalAlignment); + } +} + +void rasterizeText2D( + at::Tensor positions, + const std::vector& texts, + at::Tensor rgbBuffer, + const std::optional& color, + int textScale, + momentum::rasterizer::HorizontalAlignment horizontalAlignment, + momentum::rasterizer::VerticalAlignment verticalAlignment, + std::optional zBuffer, + const std::optional& imageOffset) { + drjit::scoped_flush_denormals flushDenorm(true); + pybind11::gil_scoped_release release; + + const int nTextBindingId = -1; + const int heightBindingId = -2003; + const int widthBindingId = -2004; + + TensorChecker checker("rasterize_text_2d"); + positions = checker.validateAndFixTensor( + positions, "positions", {nTextBindingId, 2}, {"nTexts", "xy"}, at::kFloat, true, false); + + rgbBuffer = checker.validateAndFixTensor( + rgbBuffer, + "rgb_buffer", + {heightBindingId, widthBindingId, 3}, + {"height", "width", "rgb"}, + at::kFloat, + true, + false); + + if (zBuffer.has_value()) { + zBuffer = checker.validateAndFixTensor( + *zBuffer, + "z_buffer", + {heightBindingId, widthBindingId}, + {"height", "width"}, + at::kFloat, + true, + false); + } + + const int64_t numTexts = checker.getBoundValue(nTextBindingId); + if (numTexts != static_cast(texts.size())) { + throw std::runtime_error( + fmt::format( + "Mismatch between number of positions ({}) and texts ({})", numTexts, texts.size())); + } + + for (size_t iBatch = 0; iBatch < checker.getBatchSize(); ++iBatch) { + auto rgbBufferCur = rgbBuffer.select(0, iBatch); + auto positionsCur = positions.select(0, iBatch); + auto zBufferCur = + zBuffer.has_value() ? maybeSelect(zBuffer, iBatch) : std::optional{}; + + const Eigen::Ref positionsMat = toEigenMap(positionsCur); + std::vector positionsVec; + positionsVec.reserve(positionsMat.rows()); + for (int i = 0; i < positionsMat.rows(); ++i) { + positionsVec.emplace_back(positionsMat.row(i)); + } + + momentum::rasterizer::rasterizeText2D( + positionsVec, + texts, + color.value_or(Eigen::Vector3f::Ones()), + textScale, + make_mdspan(rgbBufferCur), + make_mdspan(zBufferCur), + imageOffset.value_or(Eigen::Vector2f::Zero()), + horizontalAlignment, + verticalAlignment); + } +} + void rasterizeCircles2D( at::Tensor positions, at::Tensor rgbBuffer, diff --git a/pymomentum/renderer/software_rasterizer.h b/pymomentum/renderer/software_rasterizer.h index aefa5721cd..bb0564ca88 100644 --- a/pymomentum/renderer/software_rasterizer.h +++ b/pymomentum/renderer/software_rasterizer.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -244,6 +245,32 @@ void rasterizeLines2D( std::optional zBuffer, const std::optional& imageOffset); +void rasterizeText( + at::Tensor positions, + const std::vector& texts, + const momentum::rasterizer::Camera& camera, + at::Tensor zBuffer, + std::optional rgbBuffer, + const std::optional& color, + int textScale, + momentum::rasterizer::HorizontalAlignment horizontalAlignment, + momentum::rasterizer::VerticalAlignment verticalAlignment, + const std::optional& modelMatrix, + float nearClip, + float depthOffset, + const std::optional& imageOffset); + +void rasterizeText2D( + at::Tensor positions, + const std::vector& texts, + at::Tensor rgbBuffer, + const std::optional& color, + int textScale, + momentum::rasterizer::HorizontalAlignment horizontalAlignment, + momentum::rasterizer::VerticalAlignment verticalAlignment, + std::optional zBuffer, + const std::optional& imageOffset); + void rasterizeCircles2D( at::Tensor positions, at::Tensor rgbBuffer, From c96c60f7aca29a54ad471267ac568130930568de Mon Sep 17 00:00:00 2001 From: Chris Twigg Date: Mon, 10 Nov 2025 15:25:49 -0800 Subject: [PATCH 2/2] Refactor to remove duplicate code in marker_tracking. (#609) Summary: We have two functions, trackPosesPerframe and trackPosesForFrames, which are largely identical. This kept messing me up because I would add functionality in one of the functions or the other and then be confused why nothing had changed. Looking at the two functions, they perform the same function, we just need to pass in the appropriate set of frames into trackPosesForFrames and add support for temporal coherence. Reviewed By: cstollmeta, jeongseok-meta Differential Revision: D82688034 --- momentum/marker_tracking/marker_tracker.cpp | 278 +++++++------------- momentum/marker_tracking/marker_tracker.h | 4 +- 2 files changed, 100 insertions(+), 182 deletions(-) diff --git a/momentum/marker_tracking/marker_tracker.cpp b/momentum/marker_tracking/marker_tracker.cpp index 9098461071..e356b43da3 100644 --- a/momentum/marker_tracking/marker_tracker.cpp +++ b/momentum/marker_tracking/marker_tracker.cpp @@ -441,6 +441,23 @@ Eigen::MatrixXf trackSequence( return outMotion; } +/// Check if the global transform is zero by checking if any rigid parameters are non-zero. +/// +/// This is used to determine whether initialization is needed for pose tracking. +/// If all rigid parameters are zero, we need to solve for an initial rigid transform. +/// +/// @param dof The parameter vector to check +/// @param rigidParams Parameter set defining which parameters are rigid/global +/// @return True if global transform is zero (needs initialization), false otherwise +bool isGlobalTransformZero(const Eigen::VectorXf& dof, const ParameterSet& rigidParams) { + for (Eigen::Index i = 0; i < dof.size(); ++i) { + if (rigidParams.test(i) && dof[i] != 0.0f) { + return false; + } + } + return true; +} + /// Track poses independently per frame with fixed character identity. /// /// This is the main production tracking function used after character calibration. @@ -462,167 +479,24 @@ Eigen::MatrixXf trackPosesPerframe( const size_t frameStride) { const size_t numFrames = markerData.size(); MT_CHECK(numFrames > 0, "Input data is empty."); - MT_CHECK( - globalParams.v.size() == character.parameterTransform.numAllModelParameters(), - "Input model parameters {} do not match character model parameters {}", - globalParams.v.size(), - character.parameterTransform.numAllModelParameters()); - const ParameterTransform& pt = character.parameterTransform; - const size_t numMarkers = markerData[0].size(); - - // pose parameters need to exclude "locators" - ParameterSet poseParams = pt.getPoseParameters(); - const auto& locatorSet = pt.parameterSets.find("locators"); - if (locatorSet != pt.parameterSets.end()) { - poseParams &= ~locatorSet->second; + // Generate frame indices from stride + std::vector frameIndices; + for (size_t iFrame = 0; iFrame < numFrames; iFrame += frameStride) { + frameIndices.push_back(iFrame); } - // set up the solver - auto solverFunc = SkeletonSolverFunction(character, pt); - GaussNewtonSolverOptions solverOptions; - solverOptions.maxIterations = config.maxIter; - solverOptions.minIterations = 2; - solverOptions.doLineSearch = false; - solverOptions.verbose = config.debug; - solverOptions.threshold = 1.f; - solverOptions.regularization = config.regularization; - auto solver = GaussNewtonSolver(solverOptions, &solverFunc); - solver.setEnabledParameters(poseParams); - - // parameter limits constraint - auto limitConstrFunc = std::make_shared(character); - limitConstrFunc->setWeight(0.1); - solverFunc.addErrorFunction(limitConstrFunc); - - // positional constraint function for markers - auto posConstrFunc = std::make_shared(character, config.lossAlpha); - posConstrFunc->setWeight(PositionErrorFunction::kLegacyWeight); - solverFunc.addErrorFunction(posConstrFunc); - - std::shared_ptr skinnedLocatorPosConstrFunc = - std::make_shared(character); - skinnedLocatorPosConstrFunc->setWeight(PositionErrorFunction::kLegacyWeight); - solverFunc.addErrorFunction(skinnedLocatorPosConstrFunc); - - // floor penetration constraint data; we assume the world is y-up and floor is y=0 for mocap data. - const auto& floorConstraints = createFloorConstraints( - "Floor_", - character.locators, - Vector3f::UnitY(), - /* y offset */ 0.0f, - /* weight */ 5.0f); - auto halfPlaneConstrFunc = std::make_shared(character, /*half plane*/ true); - halfPlaneConstrFunc->setConstraints(floorConstraints); - halfPlaneConstrFunc->setWeight(PlaneErrorFunction::kLegacyWeight); - solverFunc.addErrorFunction(halfPlaneConstrFunc); - - // marker constraint data - auto constrData = createConstraintData(markerData, character.locators); - auto skinnedConstrData = createSkinnedConstraintData(markerData, character.skinnedLocators); - - // smoothness constraint only for the joints and exclude global dofs because the global transform - // needs to be accurate (may not matter in practice?) - auto smoothConstrFunc = std::make_shared( - character, poseParams & ~pt.getRigidParameters()); - smoothConstrFunc->setWeight(config.smoothing); - solverFunc.addErrorFunction(smoothConstrFunc); - - // add collision error - std::shared_ptr collisionErrorFunction; - if (config.collisionErrorWeight != 0 && character.collision != nullptr) { - collisionErrorFunction = std::make_shared(character); - collisionErrorFunction->setWeight(config.collisionErrorWeight); - solverFunc.addErrorFunction(collisionErrorFunction); + // Convert globalParams to initial motion matrix + MatrixXf initialMotion(globalParams.v.size(), numFrames); + for (size_t i = 0; i < numFrames; ++i) { + initialMotion.col(i) = globalParams.v; } - MatrixXf motion(pt.numAllModelParameters(), numFrames); - // initialize parameters to contain identity information - // the identity fields will be used but untouched during optimization - // globalParams could also be repurposed to pass in initial pose value - Eigen::VectorXf dof = globalParams.v; - size_t solverFrame = 0; - double error = 0.0; - // Use the initial global transform is it's not zero - bool needsInit = dof.head(6).isZero(0); // TODO: assume first six dofs are global dofs - - // When the frames are not continuous, we sometimes run into an issue when the desired joint - // rotation between two consecutive frames is large (eg. larger than 180). If we initialize from - // the previous result, the smaller rotation will be wrongly chosen, and we cannot recover from - // this mistake. To prevent this, we will solve each frame completely independently when they are - // not continuous. - bool continuous = (frameStride < 5); - if (!continuous) { - needsInit = true; - } + // Determine if tracking should be continuous (temporal coherence) + bool isContinuous = (frameStride < 5); - { // scope the ProgressBar so it returns - ProgressBar progress("", numFrames); - for (size_t iFrame = 0; iFrame < numFrames; iFrame += frameStride) { - // reinitialize if not continuous - if (!continuous) { - dof = globalParams.v; - } - - if ((constrData.at(iFrame).size() + skinnedConstrData.at(iFrame).size()) > - config.minVisPercent * numMarkers) { - // add positional constraints - posConstrFunc->clearConstraints(); // clear constraint data from the previous frame - posConstrFunc->setConstraints(constrData.at(iFrame)); - - skinnedLocatorPosConstrFunc->clearConstraints(); - skinnedLocatorPosConstrFunc->setConstraints(skinnedConstrData.at(iFrame)); - - // initialization - // TODO: run on first frame or tracking failure - if (needsInit) { // solve only for the rigid parameters as preprocessing - MT_LOGI_IF( - config.debug && continuous, "Solving for an initial rigid pose at frame {}", iFrame); - - // Set up different config for initialization - solverOptions.maxIterations = 50; // make sure it converges - solver.setOptions(solverOptions); - solver.setEnabledParameters(pt.getRigidParameters()); - smoothConstrFunc->setWeight(0.0); // turn off smoothing - it doesn't affect rigid dofs - - solver.solve(dof); - - // Recover solver config - solverOptions.maxIterations = config.maxIter; - solver.setOptions(solverOptions); - solver.setEnabledParameters(poseParams); - smoothConstrFunc->setWeight(config.smoothing); - - if (continuous) { - needsInit = false; - } - } - - // set smoothness target as the last pose -- dof holds parameter values from last (good) - // frame it will serve as a small regularization to rest pose for the first frame - // TODO: API needs improvement - smoothConstrFunc->setTargetParameters(dof, smoothConstrFunc->getTargetWeights()); - - error += solver.solve(dof); - ++solverFrame; - } - - // set result to output; fill in frames within a stride - // note that dof contains complete parameter info with identity - for (size_t jDelta = 0; jDelta < frameStride && iFrame + jDelta < numFrames; ++jDelta) { - motion.col(iFrame + jDelta) = dof; - } - progress.increment(frameStride); - } - } - if (config.debug) { - if (solverFrame > 0) { - MT_LOGI("Average per-frame residual: {}", error / solverFrame); - } else { - MT_LOGW("no valid frames to solve"); - } - } - return motion; + return trackPosesForFrames( + markerData, character, initialMotion, config, frameIndices, isContinuous); } /// Track poses independently for specific frame indices with fixed character identity. @@ -637,13 +511,15 @@ Eigen::MatrixXf trackPosesPerframe( /// @param initialMotion Initial parameter values (parameters x frames) /// @param config Tracking configuration settings /// @param frameIndices Vector of specific frame indices to solve +/// @param isContinuous Whether to use temporal coherence between frames /// @return Solved motion parameters matrix (parameters x frames) with poses for selected frames Eigen::MatrixXf trackPosesForFrames( const std::span> markerData, const Character& character, const MatrixXf& initialMotion, const TrackingConfig& config, - const std::vector& frameIndices) { + const std::vector& frameIndices, + bool isContinuous) { const size_t numFrames = markerData.size(); MT_CHECK(numFrames > 0, "Input data is empty."); MT_CHECK( @@ -707,6 +583,17 @@ Eigen::MatrixXf trackPosesForFrames( auto constrData = createConstraintData(markerData, character.locators); auto skinnedConstrData = createSkinnedConstraintData(markerData, character.skinnedLocators); + // smoothness constraint only for the joints and exclude global dofs because the global transform + // needs to be accurate (may not matter in practice?) + // Only use temporal smoothness if isContinuous is true + std::shared_ptr smoothConstrFunc; + if (isContinuous) { + smoothConstrFunc = std::make_shared( + character, poseParams & ~pt.getRigidParameters()); + smoothConstrFunc->setWeight(config.smoothing); + solverFunc.addErrorFunction(smoothConstrFunc); + } + // add collision error std::shared_ptr collisionErrorFunction; if (config.collisionErrorWeight != 0 && character.collision != nullptr) { @@ -718,18 +605,26 @@ Eigen::MatrixXf trackPosesForFrames( // initialize parameters to contain identity information // the identity fields will be used but untouched during optimization // globalParams could also be repurposed to pass in initial pose value - std::vector poses(frameIndices.size()); Eigen::VectorXf dof = initialMotion.col(sortedFrames.empty() ? 0 : sortedFrames[0]); size_t solverFrame = 0; double priorError = 0.0; double error = 0.0; - MatrixXf outMotion(pt.numAllModelParameters(), numFrames); + // Use the initial global transform if it's not zero + bool needsInit = isGlobalTransformZero(dof, pt.getRigidParameters()); + + MatrixXf outMotion = initialMotion; + Eigen::Index outputIndex = 0; { // scope the ProgressBar so it returns - ProgressBar progress("", sortedFrames.size()); - for (size_t fi = 0; fi < sortedFrames.size(); fi++) { - const size_t& iFrame = sortedFrames[fi]; - dof = initialMotion.col(iFrame); + ProgressBar progress("Tracking per-frame", sortedFrames.size()); + for (const auto iFrame : sortedFrames) { + // For continuous tracking, keep the solved dof from previous frame (temporal coherence) + // For non-continuous tracking, always start from initial motion (independent solving) + if (!isContinuous) { + dof = initialMotion.col(iFrame); + needsInit = true; + } + // For continuous tracking, dof is preserved from previous iteration (or initial value) if ((constrData.at(iFrame).size() + skinnedConstrData.at(iFrame).size()) > numMarkers * config.minVisPercent) { @@ -741,16 +636,37 @@ Eigen::MatrixXf trackPosesForFrames( skinnedLocatorPosConstrFunc->setConstraints(skinnedConstrData.at(iFrame)); // initialization - solverOptions.maxIterations = 50; // make sure it converges - solver.setOptions(solverOptions); - solver.setEnabledParameters(pt.getRigidParameters()); + if (needsInit) { // solve only for the rigid parameters as preprocessing + MT_LOGI_IF( + config.debug && isContinuous, + "Solving for an initial rigid pose at frame {}", + iFrame); - solver.solve(dof); + // Set up different config for initialization + solverOptions.maxIterations = 50; // make sure it converges + solver.setOptions(solverOptions); + solver.setEnabledParameters(pt.getRigidParameters()); + if (smoothConstrFunc) { + smoothConstrFunc->setWeight(0.0); // turn off smoothing - it doesn't affect rigid dofs + } + + solver.solve(dof); + + // Recover solver config + solverOptions.maxIterations = config.maxIter; + solver.setOptions(solverOptions); + solver.setEnabledParameters(poseParams); + if (smoothConstrFunc) { + smoothConstrFunc->setWeight(config.smoothing); + } - // Recover solver config - solverOptions.maxIterations = config.maxIter; - solver.setOptions(solverOptions); - solver.setEnabledParameters(poseParams); + needsInit = false; + } + + // set smoothness target as the last pose for continuous tracking + if (smoothConstrFunc) { + smoothConstrFunc->setTargetParameters(dof, smoothConstrFunc->getTargetWeights()); + } priorError += solverFunc.getError(dof); error += solver.solve(dof); @@ -758,17 +674,14 @@ Eigen::MatrixXf trackPosesForFrames( } // store result - poses[fi] = dof; + while (outputIndex <= iFrame) { + outMotion.col(outputIndex++) = dof; + } progress.increment(); } - // set results to output - size_t sortedIndex = 0; - for (size_t fi = 0; fi < numFrames; fi++) { - if (sortedIndex < sortedFrames.size() - 1 && fi == sortedFrames[sortedIndex + 1]) { - sortedIndex++; - } - outMotion.col(fi) = poses[sortedIndex]; + while (outputIndex < numFrames) { + outMotion.col(outputIndex++) = dof; } } if (config.debug) { @@ -881,7 +794,8 @@ void calibrateModel( character, motion.topRows(transform.numAllModelParameters()), trackingConfig, - firstFrame); + firstFrame, + false); // Not continuous for calibration keyframes motion.topRows(transform.numAllModelParameters()) = trackSequence( markerData, character, @@ -990,7 +904,8 @@ void calibrateModel( character, motion.topRows(transform.numAllModelParameters()), trackingConfig, - frameIndices); + frameIndices, + false); // Not continuous for calibration keyframes } else { const VectorXf initPose = motion.col(0).head(transform.numAllModelParameters()); motion.topRows(transform.numAllModelParameters()) = @@ -1119,7 +1034,8 @@ void calibrateLocators( character, motion.topRows(transform.numAllModelParameters()), trackingConfig, - frameIndices); + frameIndices, + false); // Not continuous for calibration keyframes // Solve for both markers and poses. // TODO: add a small regularization to prevent too large a change diff --git a/momentum/marker_tracking/marker_tracker.h b/momentum/marker_tracking/marker_tracker.h index acae6c139c..a10c087269 100644 --- a/momentum/marker_tracking/marker_tracker.h +++ b/momentum/marker_tracking/marker_tracker.h @@ -155,6 +155,7 @@ Eigen::MatrixXf trackPosesPerframe( /// too. /// @param[in] config Solving options. /// @param[in] frameIndices Frame indices of the frames to be solved. +/// @param[in] isContinuous Whether to use temporal coherence between frames. /// /// @return The solved motion. It has the same length as markerData. It repeats the same solved pose /// within a frame stride. @@ -163,7 +164,8 @@ Eigen::MatrixXf trackPosesForFrames( const momentum::Character& character, const Eigen::MatrixXf& initialMotion, const TrackingConfig& config, - const std::vector& frameIndices); + const std::vector& frameIndices, + bool isContinuous = false); /// Calibrate body proportions and locator offsets of a character from input marker data. ///