Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/cpp/audio-cpp/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
# rebuild and so the bump bot can see the pin.

AUDIO_CPP_VERSION?=e0f625bf5d6c801872a729ada10b3a339084eabe
AUDIO_CPP_VERSION?=86a51cab1a7e225ee87526341d52d395af8586a3
AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp

CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
Expand Down
2 changes: 2 additions & 0 deletions backend/cpp/audio-cpp/capability_routing.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ const NamedTask kTaskNames[] = {
{Task::Svc, "svc"},
{Task::Midi, "midi"},
{Task::TurnDetection, "turn"},
{Task::WakeWord, "wake"},
{Task::AudioClassification, "cls"},
};

// Accepted on input but never emitted. "spkrec" was this backend's own earlier
Expand Down
4 changes: 3 additions & 1 deletion backend/cpp/audio-cpp/capability_routing.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ enum class Task {
Svc,
Midi,
TurnDetection,
WakeWord,
AudioClassification,
};

// Mirrors engine::runtime::RunMode.
Expand Down Expand Up @@ -79,7 +81,7 @@ struct Route {
Route resolve_route(Rpc rpc, const RequestShape &shape, const Capabilities &caps);

// Canonical audio.cpp short names: gen, tts, clon, vc, svc, s2s, asr, align,
// vad, diar, sep, vdes, spk, midi, turn. parse_task_name additionally accepts "spkrec" as
// vad, diar, sep, vdes, spk, midi, turn, wake, cls. parse_task_name additionally accepts "spkrec" as
// a legacy alias; task_name only ever emits "spk".
const char *task_name(Task task);
const char *mode_name(Mode mode);
Expand Down
22 changes: 21 additions & 1 deletion backend/cpp/audio-cpp/capability_routing_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ static void test_names_round_trip() {
Task::VoiceCloning, Task::VoiceConversion,
Task::SpeechToSpeech, Task::Alignment, Task::VoiceDesign,
Task::SpeakerRecognition, Task::Svc, Task::Midi,
Task::TurnDetection};
Task::TurnDetection, Task::WakeWord, Task::AudioClassification};
for (const Task t : all) {
Task parsed = Task::Vad;
const bool ok = parse_task_name(task_name(t), parsed);
Expand Down Expand Up @@ -588,7 +588,27 @@ static void test_turn_detection_is_not_vad() {
"turn detection has an accurate capability diagnostic");
}

static void test_detection_tasks_are_not_existing_rpcs() {
for (const auto *name : {"wake", "cls"}) {
Task task = Task::Vad;
check(parse_task_name(name, task), "upstream detection task parses");
check(std::string(task_name(task)) == name, "detection task name round-trips");
Capabilities caps{"detector", {{task, {Mode::Offline}}}};
for (const auto rpc : {Rpc::Tts, Rpc::TtsStream, Rpc::AudioTranscription,
Rpc::AudioTranscriptionStream, Rpc::AudioTranscriptionLive,
Rpc::Vad, Rpc::Diarize, Rpc::SoundGeneration, Rpc::AudioTransform}) {
check(!resolve_route(rpc, RequestShape{}, caps).ok,
"detection tasks cannot serve an existing RPC");
RequestShape pinned;
pinned.pinned_task = name;
check(!resolve_route(rpc, pinned, caps).ok,
"pinning detection tasks cannot bypass RPC admission");
}
}
}

int main() {
test_detection_tasks_are_not_existing_rpcs();
test_turn_detection_is_not_vad();
test_plain_tts();
test_tts_with_voice_reference_prefers_cloning();
Expand Down
9 changes: 9 additions & 0 deletions backend/cpp/audio-cpp/loaded_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ static_assert(kMirror(Task::Svc) == 12, "Task drifted from VoiceTaskKind");
static_assert(kMirror(Task::Midi) == 13, "Task drifted from VoiceTaskKind");
static_assert(kMirror(Task::TurnDetection) == 14, "Task drifted from VoiceTaskKind");

static_assert(kEngine(engine::runtime::VoiceTaskKind::WakeWord) == 15);
static_assert(kEngine(engine::runtime::VoiceTaskKind::AudioClassification) == 16);
static_assert(kMirror(Task::WakeWord) == 15);
static_assert(kMirror(Task::AudioClassification) == 16);

static_assert(static_cast<int>(engine::runtime::RunMode::Offline) == 0, "RunMode drifted");
static_assert(static_cast<int>(engine::runtime::RunMode::Streaming) == 1,
"engine::runtime::RunMode gained, lost or reordered a member. "
Expand Down Expand Up @@ -248,6 +253,8 @@ engine::runtime::VoiceTaskKind to_engine_task(Task task) {
case Task::Svc: return K::Svc;
case Task::Midi: return K::Midi;
case Task::TurnDetection: return K::TurnDetection;
case Task::WakeWord: return K::WakeWord;
case Task::AudioClassification: return K::AudioClassification;
}
// Unreachable for any valid enumerator. No `default:` label, so -Wswitch
// still reports a member this switch stops covering.
Expand All @@ -272,6 +279,8 @@ Task from_engine_task(engine::runtime::VoiceTaskKind kind) {
case K::Svc: return Task::Svc;
case K::Midi: return Task::Midi;
case K::TurnDetection: return Task::TurnDetection;
case K::WakeWord: return Task::WakeWord;
case K::AudioClassification: return Task::AudioClassification;
}
return Task::Vad;
}
Expand Down
3 changes: 3 additions & 0 deletions docs/content/features/audio-cpp.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,9 @@ unaffected.
The upstream `turn` task performs turn detection. LocalAI has no RPC for this
task, so requests to a model that only advertises `turn` are refused. It is not
routed through voice activity detection (`vad`).
The upstream `wake` (wake-word detection) and `cls` (audio classification) tasks
are reported in capability diagnostics. They cannot serve LocalAI audio RPCs,
including VAD, even when selected with the `task` option.

Two routes are unreachable by auto-routing and need `task:` in the model config, because
nothing in a request distinguishes them from a task the same family also advertises:
Expand Down
Loading