From 2a3193468bfefcb67d72e1d6a0b199a38c38f60e Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:09:24 +0000 Subject: [PATCH 1/3] feat(ui): design voices in the voice library Generate a reference from voice instructions and sample text, then preview and save it as a reusable profile. Clear stale samples when inputs change. Assisted-by: Codex:GPT-6 Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/voice-library.spec.js | 124 ++++++++++++++++++ .../react-ui/public/locales/en/media.json | 14 +- .../react-ui/src/pages/VoiceProfileCreate.jsx | 86 +++++++++++- docs/content/features/text-to-audio.md | 13 +- 4 files changed, 229 insertions(+), 8 deletions(-) diff --git a/core/http/react-ui/e2e/voice-library.spec.js b/core/http/react-ui/e2e/voice-library.spec.js index f00e3bed0699..5149bd623c03 100644 --- a/core/http/react-ui/e2e/voice-library.spec.js +++ b/core/http/react-ui/e2e/voice-library.spec.js @@ -36,6 +36,7 @@ const profile = { async function mockVoiceAPIs(page) { const state = { createdMultipart: null } + await page.route('**/api/auth/status', route => route.fulfill({ json: { authEnabled: false } })) await page.route('**/api/models/capabilities', route => route.fulfill({ status: 200, contentType: 'application/json', @@ -216,3 +217,126 @@ test.describe('Speech voices', () => { expect(apiState.createdMultipart.readUInt16LE(wavOffset + 34)).toBe(16) }) }) + +test.describe('Voice design', () => { + let apiState + test.beforeEach(async ({ page }) => { apiState = await mockVoiceAPIs(page) }) + + async function openDesign(page) { + await page.goto('/app/voice-library/new') + await page.getByRole('button', { name: 'Design a voice', exact: true }).click() + } + + test('previews generated audio and saves its matching transcript', async ({ page }) => { + let speechRequest + await page.route('**/tts', route => { + speechRequest = route.request().postDataJSON() + return route.fulfill({ status: 200, contentType: 'audio/wav', body: pcmWav(2) }) + }) + await openDesign(page) + const generate = page.getByRole('button', { name: 'Generate reference', exact: true }) + await expect(generate).toBeDisabled() + await page.getByLabel('Voice instructions').fill(' A warm narrator. ') + await expect(generate).toBeDisabled() + await page.getByLabel('Sample text').fill(' Welcome to the documentary. ') + await generate.click() + await expect(page.getByText('Normalized reference')).toBeVisible() + expect(speechRequest).toEqual({ model: 'qwen-base', input: 'Welcome to the documentary.', instructions: 'A warm narrator.' }) + await expect(page.getByLabel('Exact transcript')).toHaveValue(speechRequest.input) + expect(apiState.createdMultipart).toBeNull() + await page.getByLabel('Voice name', { exact: true }).fill('Designed narrator') + await expect(page.getByRole('button', { name: 'Save voice', exact: true })).toBeDisabled() + await page.getByText('I confirm that this voice may be cloned').click() + await page.getByRole('button', { name: 'Save voice', exact: true }).click() + await expect(page).toHaveURL(new RegExp(`/app/voice-library\\?selected=${VOICE_ID}$`)) + expect(apiState.createdMultipart.toString()).toContain('name="transcript"\r\n\r\nWelcome to the documentary.') + const wavOffset = apiState.createdMultipart.indexOf(Buffer.from('RIFF')) + expect(wavOffset).toBeGreaterThanOrEqual(0) + expect(apiState.createdMultipart.readUInt32LE(wavOffset + 24)).toBe(24000) + expect(apiState.createdMultipart.readUInt16LE(wavOffset + 22)).toBe(1) + expect(apiState.createdMultipart.readUInt16LE(wavOffset + 34)).toBe(16) + }) + + test('requires regeneration after edits and disables save during generation', async ({ page }) => { + let releaseSpeech + await page.route('**/tts', async route => { + await new Promise(resolve => { releaseSpeech = resolve }) + await route.fulfill({ status: 200, contentType: 'audio/wav', body: pcmWav(2) }) + }) + await openDesign(page) + await page.getByLabel('Voice instructions').fill('Warm narrator') + await page.getByLabel('Sample text').fill('First sample.') + await page.getByLabel('Voice name', { exact: true }).fill('Narrator') + await page.getByText('I confirm that this voice may be cloned').click() + const generate = page.getByRole('button', { name: 'Generate reference', exact: true }) + const save = page.getByRole('button', { name: 'Save voice', exact: true }) + await generate.click() + await expect.poll(() => !!releaseSpeech).toBe(true) + await expect(save).toBeDisabled() + await expect(page.getByLabel('Sample text')).toBeDisabled() + releaseSpeech() + await expect(save).toBeEnabled() + await page.getByLabel('Sample text').fill('Second sample.') + await expect(save).toBeDisabled() + await expect(page.getByText('Normalized reference')).toBeHidden() + releaseSpeech = null + await generate.click() + await expect.poll(() => !!releaseSpeech).toBe(true) + releaseSpeech() + await expect(save).toBeEnabled() + await expect(page.getByLabel('Exact transcript')).toHaveValue('Second sample.') + await page.getByRole('group', { name: 'Voice design model', exact: true }).getByRole('button').click() + await page.getByRole('option', { name: 'piper-default', exact: true }).click() + await expect(save).toBeDisabled() + await expect(page.getByText('Normalized reference')).toBeHidden() + releaseSpeech = null + await generate.click() + await expect.poll(() => !!releaseSpeech).toBe(true) + releaseSpeech() + await expect(save).toBeEnabled() + await page.getByLabel('Voice instructions').fill('Bright narrator') + await expect(save).toBeDisabled() + await page.getByRole('button', { name: 'Upload or record', exact: true }).click() + await expect(page.locator('#voice-profile-audio-file')).toBeAttached() + await expect(save).toBeDisabled() + }) + + test('reports generation failures and permits retry', async ({ page }) => { + let attempts = 0 + await page.route('**/tts', route => { + attempts += 1 + return attempts === 1 + ? route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: { message: 'Voice design unavailable' } }) }) + : route.fulfill({ status: 200, contentType: 'audio/wav', body: pcmWav(2) }) + }) + await openDesign(page) + await page.getByLabel('Voice instructions').fill('Warm narrator') + await page.getByLabel('Sample text').fill('Try this sample.') + await page.getByRole('button', { name: 'Generate reference', exact: true }).click() + await expect(page.getByRole('alert')).toContainText('Voice design unavailable') + await expect(page.getByRole('button', { name: 'Save voice', exact: true })).toBeDisabled() + await page.getByRole('button', { name: 'Generate reference', exact: true }).click() + await expect(page.getByText('Normalized reference')).toBeVisible() + await expect(page.getByLabel('Exact transcript')).toHaveValue('Try this sample.') + }) + + test('cannot generate without an installed TTS model', async ({ page }) => { + await page.route('**/api/models/capabilities', route => route.fulfill({ json: { data: [] } })) + await openDesign(page) + await page.getByLabel('Voice instructions').fill('Warm narrator') + await page.getByLabel('Sample text').fill('Hello there.') + await expect(page.getByRole('button', { name: 'Generate reference', exact: true })).toBeDisabled() + }) + + test('rejects generated audio outside the reference duration limit', async ({ page }) => { + await page.route('**/tts', route => route.fulfill({ status: 200, contentType: 'audio/wav', body: pcmWav(0.5) })) + await openDesign(page) + await page.getByLabel('Voice instructions').fill('Warm narrator') + await page.getByLabel('Sample text').fill('Hi.') + await page.getByRole('button', { name: 'Generate reference', exact: true }).click() + await expect(page.getByRole('alert')).toContainText('Reference audio must be between 1 second and 2 minutes.') + await expect(page.getByText('Normalized reference')).toBeHidden() + await expect(page.getByRole('button', { name: 'Save voice', exact: true })).toBeDisabled() + }) + +}) diff --git a/core/http/react-ui/public/locales/en/media.json b/core/http/react-ui/public/locales/en/media.json index 614d8268deb7..8ea7b99c60d6 100644 --- a/core/http/react-ui/public/locales/en/media.json +++ b/core/http/react-ui/public/locales/en/media.json @@ -633,7 +633,7 @@ "sections": { "reference": { "title": "Reference audio", - "body": "Record in the browser or upload an existing clip. Speech should be clean, natural, and from one speaker." + "body": "Record, upload, or generate a reference clip. Speech should be clean, natural, and from one speaker." }, "details": { "title": "Identity and transcript", @@ -690,6 +690,18 @@ }, "toasts": { "created": "{{name}} is ready to use" + }, + "design": { + "source": "Reference source", + "upload": "Upload or record", + "title": "Design a voice", + "model": "Voice design model", + "hint": "Choose a TTS model that supports voice design. The list includes all TTS models; some do not support voice instructions. Describe the voice, generate a sample, and listen before saving.", + "instructions": "Voice instructions", + "text": "Sample text", + "generate": "Generate reference", + "generating": "Generating reference…", + "error": "Could not generate the voice reference." } }, "sound": { diff --git a/core/http/react-ui/src/pages/VoiceProfileCreate.jsx b/core/http/react-ui/src/pages/VoiceProfileCreate.jsx index eecf9b8301c3..af4f1876f1d1 100644 --- a/core/http/react-ui/src/pages/VoiceProfileCreate.jsx +++ b/core/http/react-ui/src/pages/VoiceProfileCreate.jsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import { Link, useNavigate, useOutletContext } from 'react-router-dom' import { useTranslation } from 'react-i18next' import LoadingSpinner from '../components/LoadingSpinner' @@ -8,7 +8,9 @@ import UnsavedChangesGuard from '../components/UnsavedChangesGuard' import ClipInput from '../components/identity/ClipInput' import WaveformPlayer from '../components/audio/WaveformPlayer' import { MAX_AUDIO_BYTES, normalizeAudioSample } from '../utils/referenceAudio' -import { voiceProfilesApi } from '../utils/api' +import { ttsApi, voiceProfilesApi } from '../utils/api' +import ModelSelector from '../components/ModelSelector' +import { CAP_TTS } from '../utils/capabilities' import Icon from '../components/Icon' function ReadinessItem({ ready, warning, children }) { @@ -25,6 +27,12 @@ export default function VoiceProfileCreate() { const { t } = useTranslation('media') const { addToast } = useOutletContext() const navigate = useNavigate() + const [referenceMode, setReferenceMode] = useState('upload') + const [designModel, setDesignModel] = useState('') + const [instructions, setInstructions] = useState('') + const [sampleText, setSampleText] = useState('') + const [generating, setGenerating] = useState(false) + const generationId = useRef(0) const [audio, setAudio] = useState(null) const [audioProcessing, setAudioProcessing] = useState(false) const [audioError, setAudioError] = useState('') @@ -40,10 +48,53 @@ export default function VoiceProfileCreate() { if (audio?.objectUrl) URL.revokeObjectURL(audio.dataUrl) }, [audio]) + useEffect(() => () => { generationId.current += 1 }, []) + + const designBusy = generating || audioProcessing || submitting + const canGenerate = !!(designModel && instructions.trim() && sampleText.trim()) && !designBusy + + const clearReference = () => { + setAudio(null) + setTranscript('') + setAudioError('') + } + + const changeDesign = (setter, value) => { + clearReference() + setter(value) + } + + const generateReference = async () => { + if (!canGenerate) return + const currentGeneration = ++generationId.current + const input = sampleText.trim() + clearReference() + setGenerating(true) + try { + const { blob } = await ttsApi.generate({ model: designModel, input, instructions: instructions.trim() }) + if (currentGeneration !== generationId.current) return + const normalized = await normalizeAudioSample({ blob, name: 'designed-reference.wav' }) + if (currentGeneration !== generationId.current) { + URL.revokeObjectURL(normalized.dataUrl) + return + } + if (normalized.duration < 1 || normalized.duration > 120) { + URL.revokeObjectURL(normalized.dataUrl) + throw new Error(t('voiceCreate.audio.durationError')) + } + setAudio(normalized) + setTranscript(input) + } catch (err) { + if (currentGeneration === generationId.current) setAudioError(err.message || t('voiceCreate.design.error')) + } finally { + if (currentGeneration === generationId.current) setGenerating(false) + } + } + const durationValid = audio?.duration >= 1 && audio?.duration <= 120 const durationRecommended = audio?.duration >= 6 && audio?.duration <= 30 const additionalReady = additionalReferences.every(reference => reference.audio && reference.transcript.trim()) - const formReady = !!audio && durationValid && name.trim() && transcript.trim() && additionalReady && consent && !audioProcessing + const formReady = !!audio && durationValid && name.trim() && transcript.trim() && additionalReady && consent && !audioProcessing && !generating const readiness = useMemo(() => ({ audio: !!audio && durationValid, @@ -127,7 +178,7 @@ export default function VoiceProfileCreate() { return (
{t('voiceCreate.title')}} @@ -142,13 +193,36 @@ export default function VoiceProfileCreate() { 01

{t('voiceCreate.sections.reference.title')}

{t('voiceCreate.sections.reference.body')}

- + + + + {referenceMode === 'design' ? ( +
+

{t('voiceCreate.design.hint')}

+
+ {t('voiceCreate.design.model')} + changeDesign(setDesignModel, value)} capability={CAP_TTS} disabled={designBusy} /> +
+
+ +