diff --git a/app/ios/Runner/AppDelegate.swift b/app/ios/Runner/AppDelegate.swift index 6e8647b32ed..c8573df453f 100644 --- a/app/ios/Runner/AppDelegate.swift +++ b/app/ios/Runner/AppDelegate.swift @@ -686,6 +686,8 @@ class SpeechRecognitionHandler: NSObject { let request = SFSpeechURLRecognitionRequest(url: fileUrl) request.shouldReportPartialResults = false request.requiresOnDeviceRecognition = true // Force on-device + request.taskHint = .dictation + request.addsPunctuation = true let task = recognizer.recognitionTask(with: request) { (recognitionResult, error) in if let error = error { diff --git a/app/lib/services/sockets/on_device_apple_provider.dart b/app/lib/services/sockets/on_device_apple_provider.dart index e803eda8682..f4a6656f08e 100644 --- a/app/lib/services/sockets/on_device_apple_provider.dart +++ b/app/lib/services/sockets/on_device_apple_provider.dart @@ -1,17 +1,19 @@ import 'dart:io'; import 'dart:typed_data'; -import 'package:flutter/services.dart'; +import 'package:flutter/services.dart' show MethodChannel; import 'package:path_provider/path_provider.dart'; import 'package:omi/models/stt_result.dart'; import 'package:omi/services/custom_stt_log_service.dart'; +import 'package:omi/services/sockets/on_device_transcript_quality_gate.dart'; import 'package:omi/services/sockets/pure_polling.dart'; class OnDeviceAppleProvider implements ISttProvider { final String language; static const MethodChannel _channel = MethodChannel('com.omi.ios/speech'); + final OnDeviceTranscriptQualityGate _qualityGate = OnDeviceTranscriptQualityGate(); OnDeviceAppleProvider({this.language = 'en'}); @@ -45,16 +47,26 @@ class OnDeviceAppleProvider implements ISttProvider { // Calculate duration: 16kHz * 2 bytes/sample * 1 channel = 32000 bytes/sec final duration = audioData.lengthInBytes / 32000.0; + final filteredText = _qualityGate.filter( + result, + audioData: audioData, + duration: Duration(microseconds: (duration * Duration.microsecondsPerSecond).round()), + ); + if (filteredText == null) { + CustomSttLogService.instance.warning('OnDeviceApple', 'Dropped low-quality local transcript: $result'); + return null; + } + CustomSttLogService.instance.info( 'OnDeviceApple', - 'Transcribed ${duration.toStringAsFixed(1)}s in ${sw.elapsedMilliseconds}ms. Text: $result', + 'Transcribed ${duration.toStringAsFixed(1)}s in ${sw.elapsedMilliseconds}ms. Text: $filteredText', ); return SttTranscriptionResult( segments: [ - SttSegment(text: result, start: audioOffsetSeconds, end: audioOffsetSeconds + duration, speakerId: 0), + SttSegment(text: filteredText, start: audioOffsetSeconds, end: audioOffsetSeconds + duration, speakerId: 0), ], - rawText: result, + rawText: filteredText, ); } finally { if (await tempFile.exists()) { diff --git a/app/lib/services/sockets/on_device_transcript_quality_gate.dart b/app/lib/services/sockets/on_device_transcript_quality_gate.dart new file mode 100644 index 00000000000..e30b0d5fbca --- /dev/null +++ b/app/lib/services/sockets/on_device_transcript_quality_gate.dart @@ -0,0 +1,74 @@ +import 'dart:typed_data'; + +class OnDeviceTranscriptQualityGate { + static const Set _fillerOnlyTokens = { + 'no', + 'nope', + 'nah', + 'uh', + 'um', + 'hmm', + }; + static const int _lowEnergyAverageAbsThreshold = 180; + + String? _lastAcceptedFiller; + + String? filter(String text, {required Uint8List audioData, required Duration duration}) { + final cleaned = _clean(text); + if (cleaned.isEmpty) return null; + + final normalized = _normalize(cleaned); + if (_isFillerOnly(normalized)) { + if (_isLowEnergyPcm(audioData)) return null; + if (_lastAcceptedFiller == normalized) return null; + _lastAcceptedFiller = normalized; + return cleaned; + } + + _lastAcceptedFiller = null; + return cleaned; + } + + static String _clean(String text) { + return text + .replaceAll(RegExp(r'\[.*?\]'), '') + .replaceAll(RegExp(r'\(.*?\)'), '') + .replaceAll(RegExp(r'\s+'), ' ') + .trim(); + } + + static String _normalize(String text) { + return text + .toLowerCase() + .replaceAll(RegExp(r"[^\p{L}\p{N}\s']+", unicode: true), '') + .replaceAll(RegExp(r'\s+'), ' ') + .trim(); + } + + static bool _isFillerOnly(String normalized) => _fillerOnlyTokens.contains(normalized); + + static bool _isLowEnergyPcm(Uint8List bytes) { + if (bytes.lengthInBytes < 2) return true; + final dataOffset = _wavDataOffset(bytes) ?? 0; + if (bytes.lengthInBytes - dataOffset < 2) return true; + + final data = ByteData.sublistView(bytes); + var totalAbs = 0; + var samples = 0; + for (var i = dataOffset; i + 1 < bytes.lengthInBytes; i += 2) { + totalAbs += data.getInt16(i, Endian.little).abs(); + samples += 1; + } + if (samples == 0) return true; + return (totalAbs / samples) < _lowEnergyAverageAbsThreshold; + } + + static int? _wavDataOffset(Uint8List bytes) { + for (var i = 12; i + 8 <= bytes.lengthInBytes; i += 1) { + if (bytes[i] == 0x64 && bytes[i + 1] == 0x61 && bytes[i + 2] == 0x74 && bytes[i + 3] == 0x61) { + return i + 8; + } + } + return null; + } +} diff --git a/app/lib/services/sockets/on_device_whisper_provider.dart b/app/lib/services/sockets/on_device_whisper_provider.dart index fc59ddbc7ef..e4b84aaa458 100644 --- a/app/lib/services/sockets/on_device_whisper_provider.dart +++ b/app/lib/services/sockets/on_device_whisper_provider.dart @@ -9,6 +9,7 @@ import 'package:whisper_flutter_new/whisper_flutter_new.dart'; import 'package:omi/models/stt_result.dart'; import 'package:omi/services/custom_stt_log_service.dart'; +import 'package:omi/services/sockets/on_device_transcript_quality_gate.dart'; import 'package:omi/services/sockets/pure_polling.dart'; class OnDeviceWhisperProvider implements ISttProvider { @@ -16,6 +17,7 @@ class OnDeviceWhisperProvider implements ISttProvider { final String language; Whisper? _whisper; bool _isInitialized = false; + final OnDeviceTranscriptQualityGate _qualityGate = OnDeviceTranscriptQualityGate(); OnDeviceWhisperProvider({required this.modelPath, this.language = 'en'}); @@ -31,19 +33,19 @@ class OnDeviceWhisperProvider implements ISttProvider { WhisperModel targetModel = WhisperModel.tiny; - if (filename.contains('tiny')) + if (filename.contains('tiny')) { targetModel = WhisperModel.tiny; - else if (filename.contains('base')) + } else if (filename.contains('base')) { targetModel = WhisperModel.base; - else if (filename.contains('small')) + } else if (filename.contains('small')) { targetModel = WhisperModel.small; - else if (filename.contains('medium')) + } else if (filename.contains('medium')) { targetModel = WhisperModel.medium; - else if (filename.contains('large-v1')) + } else if (filename.contains('large-v1')) { targetModel = WhisperModel.largeV1; - else if (filename.contains('large-v2')) + } else if (filename.contains('large-v2')) { targetModel = WhisperModel.largeV2; - else { + } else { CustomSttLogService.instance.warning( 'OnDeviceWhisper', 'Unknown model filename "$filename", defaulting to tiny.', @@ -86,29 +88,37 @@ class OnDeviceWhisperProvider implements ISttProvider { final res = await _whisper!.transcribe(transcribeRequest: req); - if (res.text == null || res.text!.isEmpty) { + if (res.text.isEmpty) { return null; } - String cleanText = res.text!.trim(); + String cleanText = res.text.trim(); cleanText = cleanText.replaceAll(RegExp(r'\[.*?\]'), '').trim(); cleanText = cleanText.replaceAll(RegExp(r'\(.*?\)'), '').trim(); - if (cleanText.isEmpty) return null; - // Calculate duration: 16kHz * 2 bytes/sample * 1 channel = 32000 bytes/sec final duration = audioData.lengthInBytes / 32000.0; + final filteredText = _qualityGate.filter( + cleanText, + audioData: audioData, + duration: Duration(microseconds: (duration * Duration.microsecondsPerSecond).round()), + ); + if (filteredText == null) { + CustomSttLogService.instance.warning('OnDeviceWhisper', 'Dropped low-quality local transcript: $cleanText'); + return null; + } + final speedFactor = sw.elapsedMilliseconds / 1000 / duration; CustomSttLogService.instance.info( 'OnDeviceWhisper', - 'Transcribed ${duration.toStringAsFixed(1)}s in ${sw.elapsedMilliseconds}ms (${speedFactor.toStringAsFixed(2)}x real-time). Text: $cleanText', + 'Transcribed ${duration.toStringAsFixed(1)}s in ${sw.elapsedMilliseconds}ms (${speedFactor.toStringAsFixed(2)}x real-time). Text: $filteredText', ); return SttTranscriptionResult( segments: [ - SttSegment(text: cleanText, start: audioOffsetSeconds, end: audioOffsetSeconds + duration, speakerId: 0), + SttSegment(text: filteredText, start: audioOffsetSeconds, end: audioOffsetSeconds + duration, speakerId: 0), ], - rawText: cleanText, + rawText: filteredText, ); } finally { if (await tempFile.exists()) { diff --git a/app/lib/services/sockets/transcription_service.dart b/app/lib/services/sockets/transcription_service.dart index e1cde2da859..b908cd0c685 100644 --- a/app/lib/services/sockets/transcription_service.dart +++ b/app/lib/services/sockets/transcription_service.dart @@ -429,8 +429,8 @@ class TranscriptSocketServiceFactory { if (Platform.isIOS) { return PurePollingSocket( config: AudioPollingConfig( - bufferDuration: const Duration(seconds: 5), - minBufferSizeBytes: sampleRate * 2, + bufferDuration: const Duration(seconds: 3), + minBufferSizeBytes: sampleRate * 2 * 2, serviceId: config.provider.name, transcoder: transcoder, ), diff --git a/app/test/unit/on_device_transcription_upgrade_test.dart b/app/test/unit/on_device_transcription_upgrade_test.dart new file mode 100644 index 00000000000..ebb1c8d0177 --- /dev/null +++ b/app/test/unit/on_device_transcription_upgrade_test.dart @@ -0,0 +1,115 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:omi/services/sockets/on_device_apple_provider.dart'; +import 'package:omi/services/sockets/on_device_transcript_quality_gate.dart'; + +void main() { + group('OnDeviceTranscriptQualityGate', () { + test('drops repeated one-word no hallucinations', () { + final gate = OnDeviceTranscriptQualityGate(); + + expect(gate.filter('no', audioData: _loudPcmWav(), duration: const Duration(seconds: 3)), 'no'); + expect(gate.filter('No.', audioData: _loudPcmWav(), duration: const Duration(seconds: 3)), isNull); + expect(gate.filter(' no ', audioData: _loudPcmWav(), duration: const Duration(seconds: 3)), isNull); + }); + + test('drops filler no on low-energy audio', () { + final gate = OnDeviceTranscriptQualityGate(); + + expect(gate.filter('no', audioData: _silentPcmWav(), duration: const Duration(seconds: 3)), isNull); + }); + + test('keeps real speech and resets duplicate gate', () { + final gate = OnDeviceTranscriptQualityGate(); + + expect(gate.filter('no', audioData: _loudPcmWav(), duration: const Duration(seconds: 3)), 'no'); + expect(gate.filter('no problem, start capture', audioData: _loudPcmWav(), duration: const Duration(seconds: 3)), + 'no problem, start capture'); + expect(gate.filter('no', audioData: _loudPcmWav(), duration: const Duration(seconds: 3)), 'no'); + }); + }); + + test('iOS Apple on-device STT uses a multi-second polling window', () { + final source = File('lib/services/sockets/transcription_service.dart').readAsStringSync(); + final appleBlock = RegExp( + r'if \(Platform\.isIOS\) \{(?[\s\S]*?)sttProvider: OnDeviceAppleProvider', + multiLine: true, + ).firstMatch(source)!.namedGroup('block')!; + + expect(appleBlock, contains('bufferDuration: const Duration(seconds: 3)')); + expect(appleBlock, contains('minBufferSizeBytes: sampleRate * 2 * 2')); + }); + + test('native iOS speech fallback is tuned for dictation', () { + final source = File('ios/Runner/AppDelegate.swift').readAsStringSync(); + + expect(source, contains('request.taskHint = .dictation')); + expect(source, contains('request.addsPunctuation = true')); + }); + + test('Apple on-device STT drops low-energy filler transcripts', () async { + TestWidgetsFlutterBinding.ensureInitialized(); + final tempDir = await Directory.systemTemp.createTemp('on-device-apple-provider-'); + const pathProviderChannel = MethodChannel('plugins.flutter.io/path_provider'); + const speechChannel = MethodChannel('com.omi.ios/speech'); + + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + pathProviderChannel, + (call) async => call.method == 'getTemporaryDirectory' ? tempDir.path : null, + ); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + speechChannel, + (call) async => 'no', + ); + + try { + final provider = OnDeviceAppleProvider(); + + final result = await provider.transcribe(_silentPcmWav()); + + expect(result, isNull); + } finally { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + pathProviderChannel, + null, + ); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + speechChannel, + null, + ); + if (await tempDir.exists()) await tempDir.delete(recursive: true); + } + }); +} + +Uint8List _silentPcmWav() => _pcmWav(List.filled(1600, 0)); + +Uint8List _loudPcmWav() => _pcmWav(List.generate(1600, (i) => i.isEven ? 9000 : -9000)); + +Uint8List _pcmWav(List samples) { + final dataSize = samples.length * 2; + final out = BytesBuilder(); + void ascii(String value) => out.add(value.codeUnits); + void u16(int value) => out.add([value & 0xff, (value >> 8) & 0xff]); + void u32(int value) => out.add([value & 0xff, (value >> 8) & 0xff, (value >> 16) & 0xff, (value >> 24) & 0xff]); + ascii('RIFF'); + u32(36 + dataSize); + ascii('WAVEfmt '); + u32(16); + u16(1); + u16(1); + u32(16000); + u32(32000); + u16(2); + u16(16); + ascii('data'); + u32(dataSize); + for (final sample in samples) { + final value = sample < 0 ? 0x10000 + sample : sample; + u16(value); + } + return out.toBytes(); +}