diff --git a/__tests__/pro/locketPlayerSmoke.test.tsx b/__tests__/pro/locketPlayerSmoke.test.tsx deleted file mode 100644 index 1cbeba0a6..000000000 --- a/__tests__/pro/locketPlayerSmoke.test.tsx +++ /dev/null @@ -1,105 +0,0 @@ -/** - * LocketPlayerScreen smoke (rendered integration) — behaviour-neutrality guard for the - * component decomposition (docs/plans/locket-screens-refactor.md phases B + D) and the - * usePlayerScreen hook extraction. - * - * Mounts the REAL LocketPlayerScreen on a REAL native stack (fakes only at the device boundary; - * no mocking our own code), seeds recordings via the store's real writers, and asserts the real - * affordances render. The transcript/summary cases drive the logic that moved into the hook - * (recording lookup, transcriptComplete/hasPartial derivations, speakerRows, summary gating), - * so this guard actually covers the extraction — green before and after = no behaviour change. - */ -// Un-mock react-navigation for THIS file: jest.setup stubs it globally (navigate no-op, useRoute {}), -// which breaks real route params. requireActual restores the real library so the seeded recordingId -// reaches the screen. -jest.mock('@react-navigation/native', () => jest.requireActual('@react-navigation/native')); - -import { installNativeBoundary, requireRTL } from '../harness/nativeBoundary'; -import { installPro } from '../harness/proHarness'; - -// Seed one recording (optionally enriched with transcript/summary via the real store writer), -// then mount the REAL player screen on a REAL native stack routed to it. Returns the RTL result. -async function renderSeededPlayer(enrich?: Record) { - const boundary = installNativeBoundary({ fs: true }); - const React = require('react'); - const { render, act } = requireRTL(); - await installPro(); - const { NavigationContainer } = require('@react-navigation/native'); - const { createNativeStackNavigator } = require('@react-navigation/native-stack'); - const { getRegisteredScreens } = require('../../src/navigation/screenRegistry'); - const { useRecordingsStore } = require('@offgrid/pro/locket/stores'); - - useRecordingsStore.setState({ recordings: [], jobs: [] }); - - const NOW = Date.now(); - boundary.fs!.seedFile('/docs/rec.wav', 5_000_000); - useRecordingsStore.getState().addFinalized({ - path: '/docs/rec.wav', startedAt: NOW, endedAt: NOW, - durationMs: 30 * 60 * 1000, sizeBytes: 5_000_000, - }); - const id: string = useRecordingsStore.getState().recordings[0].id; - if (enrich) useRecordingsStore.getState().updateRecording(id, enrich); - - const Stack = createNativeStackNavigator(); - const screens = getRegisteredScreens(); - const App = () => - React.createElement(NavigationContainer, null, - React.createElement(Stack.Navigator, - { initialRouteName: 'LocketPlayer', screenOptions: { headerShown: false } }, - ...screens.map((sc: { name: string; component: React.ComponentType }) => - React.createElement(Stack.Screen, { - key: sc.name, - name: sc.name, - component: sc.component, - initialParams: sc.name === 'LocketPlayer' ? { sessionId: 'sess', recordingId: id } : undefined, - })), - )); - const ui = render(React.createElement(App)); - // Let the screen mount + the player hook settle (the audio boundary is faked). - await act(async () => { await new Promise((r) => setTimeout(r, 300)); }); - return ui; -} - -describe('LocketPlayerScreen smoke (rendered): mounts a seeded recording', () => { - it('renders the player for a seeded recording', async () => { - const ui = await renderSeededPlayer(); - // The header overflow + info controls are present whenever the recording is found — - // a stable anchor that does not depend on transient player-load state. - expect(ui.getByTestId('recording-menu')).toBeTruthy(); - expect(ui.getByTestId('recording-info')).toBeTruthy(); - }); - - it('shows the transcribe affordances for a completed, un-summarized transcript', async () => { - // transcriptStatus 'done' => transcriptComplete true, hasPartial false: the finished-transcript - // action row (Summarize / Add to chat) and the transcript re-run control render. - const ui = await renderSeededPlayer({ - transcript: 'hello there. how are you.', - transcriptStatus: 'done', - transcriptSegments: [ - { text: 'hello there.', startMs: 0, endMs: 1500 }, - { text: 'how are you.', startMs: 1500, endMs: 3000 }, - ], - }); - expect(ui.getByTestId('summarize-recording')).toBeTruthy(); - expect(ui.getByTestId('add-to-chat')).toBeTruthy(); - expect(ui.getByTestId('re-transcribe')).toBeTruthy(); - }); - - it('shows the summary affordances once a transcript is summarized', async () => { - // With a stored summary the Summary section renders (toggle + re-summarize) and the - // Summarize CTA is replaced — the exact gating the extracted hook now derives. - const ui = await renderSeededPlayer({ - transcript: 'hello there. how are you.', - transcriptStatus: 'done', - transcriptSegments: [ - { text: 'hello there.', startMs: 0, endMs: 1500 }, - { text: 'how are you.', startMs: 1500, endMs: 3000 }, - ], - summary: '- greeting exchanged\n- wellbeing check', - summaryStatus: 'done', - }); - expect(ui.getByTestId('summary-toggle')).toBeTruthy(); - expect(ui.getByTestId('re-summarize')).toBeTruthy(); - expect(ui.getByTestId('re-transcribe')).toBeTruthy(); - }); -}); diff --git a/__tests__/pro/locketSpeakerTurns.test.tsx b/__tests__/pro/locketSpeakerTurns.test.tsx new file mode 100644 index 000000000..34dc58a03 --- /dev/null +++ b/__tests__/pro/locketSpeakerTurns.test.tsx @@ -0,0 +1,84 @@ +/** + * Speaker turns render on the recording DETAIL (rendered integration) — the Locket recorder. + * + * tdrz transcription marks voice changes with a [SPEAKER_TURN] token in the segment text. The + * detail screen must turn those into visible "Speaker 1 / Speaker 2" labels above the transcript + * rows. This is the screen search + a normal tap both land on (the old LocketPlayer that used to + * render turns was removed and its mapping ported here), so this guards that the diarization + * actually shows where the user reaches it. + * + * Real detail screen on a real nav stack; the recording (analysed, with tdrz segments) is seeded + * via the store's real writers (the device-leaf). Parameterized so both branches falsify: with + * [SPEAKER_TURN] markers the labels appear; without them, no "Speaker" label renders. + * + * Placement: __tests__/pro/ (the canonical home for pro-importing rendered tests). + */ +jest.mock('@react-navigation/native', () => jest.requireActual('@react-navigation/native')); + +import { installNativeBoundary, requireRTL } from '../harness/nativeBoundary'; +import { installPro } from '../harness/proHarness'; + +const CASES: { name: string; withTurns: boolean }[] = [ + { name: 'tdrz [SPEAKER_TURN] markers → Speaker 1/2 labels', withTurns: true }, + { name: 'no markers → no speaker labels', withTurns: false }, +]; + +describe('speaker turns on the recording detail (rendered)', () => { + it.each(CASES)('$name', async ({ withTurns }) => { + const boundary = installNativeBoundary({ fs: true }); + /* eslint-disable @typescript-eslint/no-var-requires */ + const React = require('react'); + const { render, fireEvent, waitFor, act } = requireRTL(); + await installPro(); + const { NavigationContainer } = require('@react-navigation/native'); + const { createNativeStackNavigator } = require('@react-navigation/native-stack'); + const { getRegisteredScreens } = require('../../src/navigation/screenRegistry'); + const { useRecordingsStore } = require('@offgrid/pro/locket/stores'); + /* eslint-enable @typescript-eslint/no-var-requires */ + + useRecordingsStore.setState({ recordings: [], jobs: [] }); + const NOW = Date.now(); + boundary.fs!.seedFile('/docs/rec.wav', 5_000_000); + useRecordingsStore.getState().addFinalized({ + path: '/docs/rec.wav', startedAt: NOW, endedAt: NOW, + durationMs: 30 * 60 * 1000, sizeBytes: 5_000_000, + }); + const id: string = useRecordingsStore.getState().recordings[0].id; + // A [SPEAKER_TURN] on the middle segment flips the speaker, so the first row reads "Speaker 1" + // and the last "Speaker 2". Analysed (on-device) so the detail shows its transcript body rather + // than the Analyse CTA. The stripped segment text stays non-empty so the rows render. + const turn = withTurns ? ' [SPEAKER_TURN]' : ''; + useRecordingsStore.getState().updateRecording(id, { + transcript: 'alpha. beta. gamma.', + transcriptStatus: 'done', transcribedAt: NOW, prunedAt: NOW, + transcriptSegments: [ + { text: 'alpha.', startMs: 0, endMs: 1000 }, + { text: `beta.${turn}`, startMs: 1000, endMs: 2000 }, + { text: 'gamma.', startMs: 2000, endMs: 3000 }, + ], + insightsSource: 'on-device', insightsAt: 1, title: 'T', summary: 'S', keyPoints: ['k'], + actionItems: [{ id: 'a', text: 'x' }], + } as never); + + const Stack = createNativeStackNavigator(); + const screens = getRegisteredScreens(); + const App = () => React.createElement(NavigationContainer, null, + React.createElement(Stack.Navigator, { initialRouteName: 'LocketRecording', screenOptions: { headerShown: false } }, + ...screens.map((sc: { name: string; component: React.ComponentType }) => + React.createElement(Stack.Screen, { + key: sc.name, name: sc.name, component: sc.component, + initialParams: sc.name === 'LocketRecording' ? { recordingId: id } : undefined, + })))); + const ui = render(React.createElement(App)); + await act(async () => { await new Promise((r) => setTimeout(r, 0)); }); + + // Transcript is collapsed by default — expand it so the segment rows (+ speaker labels) render. + await waitFor(() => ui.getByTestId('insights-toggle-transcript')); + await act(async () => { fireEvent.press(ui.getByTestId('insights-toggle-transcript')); await Promise.resolve(); }); + await waitFor(() => ui.getByTestId('transcript-seg-0')); + + // The terminal artifact: visible "Speaker N" labels iff the tdrz markers were present. + expect(ui.queryByText('Speaker 1') != null).toBe(withTurns); + expect(ui.queryByText('Speaker 2') != null).toBe(withTurns); + }); +}); diff --git a/__tests__/pro/ui/LocketRecordingRepair.test.tsx b/__tests__/pro/ui/LocketRecordingRepair.test.tsx new file mode 100644 index 000000000..7ffaecf18 --- /dev/null +++ b/__tests__/pro/ui/LocketRecordingRepair.test.tsx @@ -0,0 +1,123 @@ +/** + * LocketRecordingScreen - damaged-clip Repair (Pro) integration test. + * + * Drives the REAL LocketRecordingScreen + REAL useRecordingDetail + REAL + * repairRecording service against the REAL recordings store. Fakes ONLY device + * boundaries: the vector-icon shims, theme, navigation, toast, and the native + * AudioNormalizer.repairWavHeader / RecordingPlayer modules. Asserts what the + * user sees: a recovered clip with a stale header shows a Repair banner, and + * tapping Repair clears it (the header got rewritten -> needsRepair false). + * + * Skips in open-core CI where pro/ is absent. + */ + +import React from 'react'; +import { NativeModules } from 'react-native'; +import { render, fireEvent, waitFor } from '@testing-library/react-native'; + +const mockColors = { + text: '#000', textMuted: '#999', textSecondary: '#666', textDisabled: '#bbb', + primary: '#1DB954', error: '#E00', trending: '#F90', + background: '#FFF', surface: '#F5F5F5', surfaceLight: '#EEE', border: '#E0E0E0', overlay: 'rgba(0,0,0,0.4)', +}; + +jest.mock('react-native-vector-icons/Feather', () => () => null); +jest.mock('react-native-vector-icons/MaterialIcons', () => () => null); + +const mockShadows = new Proxy({}, { get: () => ({}) }); +jest.mock('@offgrid/core/theme', () => { + const actual = jest.requireActual('@offgrid/core/theme'); + return { + ...actual, + useTheme: () => ({ colors: mockColors, shadows: mockShadows, isDark: false }), + // Real useThemedStyles calls factory(colors, shadows) - mirror that here. + useThemedStyles: (fn: (c: typeof mockColors, s: unknown) => unknown) => fn(mockColors, mockShadows), + }; +}); + +jest.mock('@offgrid/core/utils/toast', () => ({ showToast: jest.fn() })); + +jest.mock('@react-navigation/native', () => { + const actual = jest.requireActual('@react-navigation/native'); + return { + ...actual, + useNavigation: () => ({ navigate: jest.fn(), goBack: jest.fn() }), + useRoute: () => ({ params: { recordingId: 'rec-1' } }), + }; +}); + +// Native boundary: the header repair + the player module the detail screen mounts. +const repairMock = jest.fn().mockResolvedValue({ alreadyHealthy: false, durationMs: 5000, sizeBytes: 100000 }); +(NativeModules as unknown as Record).AudioNormalizer = { + repairWavHeader: repairMock, + normalizeToWav16kMono: jest.fn().mockResolvedValue('/tmp/x.wav'), +}; +(NativeModules as unknown as Record).RecordingPlayer = { + addListener: jest.fn(), + removeListeners: jest.fn(), + load: jest.fn().mockResolvedValue(undefined), + play: jest.fn(), + pause: jest.fn(), + seek: jest.fn(), + stop: jest.fn(), +}; + +type ScreenModule = typeof import('@offgrid/pro/locket/screens/LocketRecordingScreen'); +type StoreModule = typeof import('@offgrid/pro/locket/stores'); + +function load(): { screen: ScreenModule; store: StoreModule } | null { + try { + return { + screen: require(['..', '..', '..', 'pro', 'locket', 'screens', 'LocketRecordingScreen'].join('/')), + store: require(['..', '..', '..', 'pro', 'locket', 'stores'].join('/')), + }; + } catch { + return null; + } +} + +const mods = load(); +const maybe = mods ? describe : describe.skip; + +maybe('LocketRecordingScreen - damaged clip repair', () => { + const { LocketRecordingScreen } = mods!.screen; + const { useRecordingsStore } = mods!.store; + const initial = useRecordingsStore.getState(); + const now = Date.now(); + + const damaged = { + id: 'rec-1', + path: '/tmp/rec-1.wav', + startedAt: now - 60_000, + endedAt: now, + durationMs: 60_000, + sizeBytes: 500_000, + prunedAt: now, + name: 'Recovered (header damaged)', + // The recovered-with-stale-header signal set by recovery. + needsRepair: true, + }; + + beforeEach(() => { + repairMock.mockClear(); + useRecordingsStore.setState({ recordings: [damaged] }); + }); + afterEach(() => { + useRecordingsStore.setState(initial, true); + }); + + it('shows the Repair banner for a damaged clip and clears it after repair', async () => { + const { getByTestId, queryByTestId, getByText } = render(); + + // The user sees the banner - the clip won't play until the header is fixed. + expect(getByTestId('repair-banner')).toBeTruthy(); + expect(getByText(/Recovered after the app closed/i)).toBeTruthy(); + + // Tapping Repair dispatches the real repairRecording service (native header fix). + fireEvent.press(getByTestId('repair-header')); + await waitFor(() => expect(repairMock).toHaveBeenCalledWith('/tmp/rec-1.wav')); + + // Header rewritten -> needsRepair cleared -> the banner is gone. + await waitFor(() => expect(queryByTestId('repair-banner')).toBeNull()); + }); +}); diff --git a/__tests__/unit/services/whisperService.test.ts b/__tests__/unit/services/whisperService.test.ts index abdc6350f..cf87d33c7 100644 --- a/__tests__/unit/services/whisperService.test.ts +++ b/__tests__/unit/services/whisperService.test.ts @@ -347,10 +347,10 @@ describe('WhisperService', () => { const mockContext = { id: 'ctx', release: jest.fn(), transcribeRealtime: jest.fn(), transcribe: jest.fn() }; mockedInitWhisper.mockResolvedValue(mockContext as any); - await whisperService.loadModel('/path/to/model.bin', { useCoreML: true, useGpu: true, useFlashAttn: true }); + await whisperService.loadModel('/path/to/model.bin', { useCoreML: true }); expect(initWhisper).toHaveBeenCalledWith( - expect.objectContaining({ useCoreMLIos: true, useGpu: true, useFlashAttn: true }), + expect.objectContaining({ useCoreMLIos: true }), ); }); diff --git a/patches/react-native-calendar-events+2.2.0.patch b/patches/react-native-calendar-events+2.2.0.patch new file mode 100644 index 000000000..7e0bbf10f --- /dev/null +++ b/patches/react-native-calendar-events+2.2.0.patch @@ -0,0 +1,30 @@ +diff --git a/node_modules/react-native-calendar-events/ios/RNCalendarEvents.m b/node_modules/react-native-calendar-events/ios/RNCalendarEvents.m +index a85c013..e36e344 100644 +--- a/node_modules/react-native-calendar-events/ios/RNCalendarEvents.m ++++ b/node_modules/react-native-calendar-events/ios/RNCalendarEvents.m +@@ -761,14 +761,23 @@ - (NSDictionary *)serializeCalendarEvent:(EKEvent *)event + + RCT_EXPORT_METHOD(requestPermissions:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) + { +- [self.eventStore requestAccessToEntityType:EKEntityTypeEvent completion:^(BOOL granted, NSError *error) { ++ void (^handler)(BOOL, NSError *) = ^(BOOL granted, NSError *error) { + NSString *status = granted ? @"authorized" : @"denied"; + if (!error) { + resolve(status); + } else { + reject(@"error", @"authorization request error", error); + } +- }]; ++ }; ++ // iOS 17+ made requestAccessToEntityType "no longer allowed" for reads; reading ++ // events requires the new full-access API (requestFullAccessToEventsWithCompletion). ++ // Fall back to the old API on iOS 16 and earlier. Patched here because ++ // react-native-calendar-events is unmaintained (latest 2.2.0, Jan 2021, predates iOS 17). ++ if (@available(iOS 17.0, *)) { ++ [self.eventStore requestFullAccessToEventsWithCompletion:handler]; ++ } else { ++ [self.eventStore requestAccessToEntityType:EKEntityTypeEvent completion:handler]; ++ } + } + + RCT_EXPORT_METHOD(findCalendars:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) diff --git a/src/components/ModelCard.tsx b/src/components/ModelCard.tsx index 86c951f97..3f49dbd86 100644 --- a/src/components/ModelCard.tsx +++ b/src/components/ModelCard.tsx @@ -58,6 +58,13 @@ interface ModelCardProps { recommended?: RecommendedConfig; /** Model can run on the GPU/NPU (LiteRT or Q4_0/Q8_0 GGUF) → show the badge. */ supportsAcceleration?: boolean; + /** + * iOS whisper only: state of this model's CoreML (Neural Engine) encoder. + * 'ready' = encoder present & valid (runs on the ANE); 'unavailable' = downloaded but + * no valid encoder (runs on CPU). Undefined = don't show (non-iOS, not downloaded, + * or not a whisper model). Additive/optional so only the Transcription tab renders it. + */ + coreMLStatus?: 'ready' | 'downloading' | 'unavailable'; failedState?: { errorMessage: string; bytesDownloaded: number; @@ -186,6 +193,7 @@ export const ModelCard: React.FC = ({ isTrending, recommended, supportsAcceleration, + coreMLStatus, failedState, }) => { const styles = useThemedStyles(createStyles); @@ -234,6 +242,7 @@ export const ModelCard: React.FC = ({ isTrending={isTrending} recommended={recommended} supportsAcceleration={supportsAcceleration} + coreMLStatus={coreMLStatus} /> ) : ( = ({ isActive={isActive} recommended={recommended} supportsAcceleration={supportsAcceleration} + coreMLStatus={coreMLStatus} /> )} diff --git a/src/components/ModelCardContent.tsx b/src/components/ModelCardContent.tsx index a5c5c1862..3ddb98235 100644 --- a/src/components/ModelCardContent.tsx +++ b/src/components/ModelCardContent.tsx @@ -9,6 +9,37 @@ import { huggingFaceService } from '../services/huggingface'; import { ModelCredibility } from '../types'; import { triggerHaptic } from '../utils/haptics'; +/** iOS whisper CoreML encoder badge: 'ready' → ANE (accent), 'unavailable' → CPU + * (muted). Extracted so the card content stays simple. Renders nothing otherwise. */ +const CoreMLBadge: React.FC<{ + status?: 'ready' | 'downloading' | 'unavailable'; + styles: ReturnType; +}> = ({ status, styles }) => { + if (status === 'ready') { + return ( + + ANE + + ); + } + if (status === 'unavailable') { + return ( + + CPU + + ); + } + return null; +}; + +/** Whether the compact card's badge row has anything to show. Kept out of the render so + * the OR-chain doesn't inflate the component's cyclomatic complexity. */ +const hasCompactBadges = ( + hasTypeOrParams: boolean, + supportsAcceleration?: boolean, + coreMLStatus?: string, +): boolean => hasTypeOrParams || !!supportsAcceleration || !!coreMLStatus; + interface CredibilityInfo { color: string; label: string; @@ -56,6 +87,8 @@ interface CompactModelCardContentProps { recommended?: RecommendedConfig; /** Model can run on the GPU/NPU (LiteRT or Q4_0/Q8_0 GGUF) → show the badge. */ supportsAcceleration?: boolean; + /** iOS whisper CoreML encoder state: 'ready' (ANE), 'unavailable' (CPU). Optional. */ + coreMLStatus?: 'ready' | 'downloading' | 'unavailable'; } function formatNumber(num: number): string { @@ -97,6 +130,7 @@ export const CompactModelCardContent: React.FC = ( isTrending, recommended, supportsAcceleration, + coreMLStatus, }) => { const { colors } = useTheme(); const styles = useThemedStyles(createStyles); @@ -150,7 +184,7 @@ export const CompactModelCardContent: React.FC = ( ))} - ) : (model.modelType || model.paramCount || supportsAcceleration) && ( + ) : hasCompactBadges(!!model.modelType || !!model.paramCount, supportsAcceleration, coreMLStatus) && ( {/* Capability badge: this model can run on the GPU/NPU (a LiteRT model or a Q4_0/Q8_0 GGUF). K-quants silently fall back to CPU, so they get no badge. */} @@ -159,6 +193,8 @@ export const CompactModelCardContent: React.FC = ( NPU/GPU )} + {/* iOS whisper: Neural Engine (ANE) ready vs CPU-only, per model. */} + {model.modelType && ( @@ -196,6 +232,8 @@ interface StandardModelCardContentProps { recommended?: RecommendedConfig; /** Model can run on the GPU/NPU (LiteRT or Q4_0/Q8_0 GGUF) → show the badge. */ supportsAcceleration?: boolean; + /** iOS whisper CoreML encoder state: 'ready' (ANE), 'unavailable' (CPU). Optional. */ + coreMLStatus?: 'ready' | 'downloading' | 'unavailable'; } export const StandardModelCardContent: React.FC = ({ @@ -205,6 +243,7 @@ export const StandardModelCardContent: React.FC = isActive, recommended, supportsAcceleration, + coreMLStatus, }) => { const { colors } = useTheme(); const styles = useThemedStyles(createStyles); @@ -252,6 +291,8 @@ export const StandardModelCardContent: React.FC = NPU/GPU )} + {/* iOS whisper: Neural Engine (ANE) ready vs CPU-only, per model. */} + {!!description && ( diff --git a/src/screens/ModelsScreen/TranscriptionModelsTab.tsx b/src/screens/ModelsScreen/TranscriptionModelsTab.tsx index 6b775e819..806fb2955 100644 --- a/src/screens/ModelsScreen/TranscriptionModelsTab.tsx +++ b/src/screens/ModelsScreen/TranscriptionModelsTab.tsx @@ -11,7 +11,7 @@ * the active one. */ import React, { useCallback, useEffect, useState } from 'react'; -import { View, Text, ScrollView, TouchableOpacity } from 'react-native'; +import { View, Text, ScrollView, TouchableOpacity, Platform } from 'react-native'; import { useFocusEffect } from '@react-navigation/native'; import Icon from 'react-native-vector-icons/Feather'; import { ModelCard } from '../../components'; @@ -21,7 +21,7 @@ import type { ThemeColors, ThemeShadows } from '../../theme'; import { TYPOGRAPHY, SPACING } from '../../constants'; import { useWhisperStore } from '../../stores'; import { useSttDownloadState } from '../../hooks/useSttDownloadState'; -import { WHISPER_MODELS } from '../../services'; +import { WHISPER_MODELS, whisperService } from '../../services'; import { createStyles as createModelsScreenStyles } from './styles'; import logger from '../../utils/logger'; @@ -48,6 +48,21 @@ const WhisperCard: React.FC = ({ }) => { const present = presentModelIds.includes(model.id); const active = downloadedModelId === model.id; + // iOS only: is this downloaded model's CoreML (Neural Engine) encoder present & valid? + // Drives the ANE/CPU badge so users can see which models run on the Neural Engine. + const [coreMLStatus, setCoreMLStatus] = useState<'ready' | 'unavailable' | undefined>(undefined); + useEffect(() => { + if (Platform.OS !== 'ios' || !present || !model.coreMLUrl) { + setCoreMLStatus(undefined); + return; + } + let cancelled = false; + whisperService + .hasCoreMLEncoder(model.id) + .then((ok) => { if (!cancelled) setCoreMLStatus(ok ? 'ready' : 'unavailable'); }) + .catch(() => { if (!cancelled) setCoreMLStatus('unavailable'); }); + return () => { cancelled = true; }; + }, [present, model.id, model.coreMLUrl]); // WHISPER_MODELS sizes are in MB. Surface bytes so the STT card matches the // Text/Image cards ("X MB / Y MB"); for a queued model this reads "0 B / 142 MB". const totalBytes = model.size * 1024 * 1024; @@ -64,6 +79,7 @@ const WhisperCard: React.FC = ({ isQueued={queued} downloadProgress={downloadProgress} downloadBytes={downloadBytes} + coreMLStatus={coreMLStatus} testID={`transcription-model-card-${index}`} // Present but not active → tap to use; not present → tap to download. onPress={downloading ? undefined : (present ? (active ? undefined : () => onSelect(model.id)) : () => onDownload(model.id))} diff --git a/src/services/hardware.ts b/src/services/hardware.ts index fabf50c97..95cd5c102 100644 --- a/src/services/hardware.ts +++ b/src/services/hardware.ts @@ -60,6 +60,16 @@ class HardwareService { logger.log(`[WIRE-DEVICE] ${JSON.stringify({ platform: Platform.OS, ...this.cachedDeviceInfo })}`); // [WIRE] real device caps (drives onboarding recs + memory budget) return this.cachedDeviceInfo; } + + /** + * Single definition of "can this device safely offload to the GPU (Metal)". GPU offload + * is iOS-only (Metal), skips emulators (they misreport GPU), and needs >4GB RAM (Metal + * can OOM 4GB devices). Reused by the whisper load path, the whisper settings UI, and the + * LLM vision path so the rule lives in ONE place, not copied per caller. + */ + deviceSupportsGpuOffload(info: DeviceInfoType): boolean { + return Platform.OS === 'ios' && !info.isEmulator && info.totalMemory > 4 * 1024 * 1024 * 1024; + } /** * Real free memory the system can hand out RIGHT NOW. On Android this reads * `MemAvailable` from /proc/meminfo (what the kernel will give without @@ -451,6 +461,20 @@ class HardwareService { return { hasNpu: HTP_ENABLED && soc.hasNPU, hasGpu: opencl.supported }; } + /** + * Whisper GPU-offload eligibility, cross-platform and authoritative — the ONE predicate + * both the whisper load site and the whisper settings toggle read, so they always agree. + * iOS: Metal is safe on a real device with >4GB RAM (deviceSupportsGpuOffload). + * Android: the ggml OpenCL backend (ported into whisper.rn) needs a compatible GPU + * (Adreno/Mali, OpenCL 3.0), probed via getAccelerationCapability().hasGpu. + * This is whisper-specific and does NOT change the iOS/LLM GPU rules. + */ + async whisperSupportsGpu(): Promise { + if (Platform.OS === 'ios') return this.deviceSupportsGpuOffload(await this.getDeviceInfo()); + if (Platform.OS === 'android') return (await this.getAccelerationCapability()).hasGpu; + return false; + } + async getOpenCLCapability(): Promise<{ supported: boolean; reason?: string }> { if (this.cachedOpenCLCapability) return this.cachedOpenCLCapability; if (Platform.OS !== 'android') return { supported: false, reason: 'not_android' }; diff --git a/src/services/transcriptSummarizer.ts b/src/services/transcriptSummarizer.ts index ae3f111a3..86ce9e4ed 100644 --- a/src/services/transcriptSummarizer.ts +++ b/src/services/transcriptSummarizer.ts @@ -234,6 +234,16 @@ class TranscriptSummarizerService { return llmService.isModelLoaded() || isLiteRTActive() || isRemoteActive(); } + /** + * True when a remote provider is connected, so summaries can run WITHOUT a local + * model download. Exposed so callers (e.g. the recorder's model-readiness guard) + * decide "is a summary backend available" from the SAME signal the summarizer + * uses to route - never a second copy that could drift. + */ + hasRemoteBackend(): boolean { + return isRemoteActive(); + } + /** Subscribe to progress. The listener is not called with a current value. */ subscribe(listener: (p: SummarizeProgress) => void): () => void { this.listeners.add(listener); diff --git a/src/services/whisperService.ts b/src/services/whisperService.ts index 675248aa5..ae6ec4391 100644 --- a/src/services/whisperService.ts +++ b/src/services/whisperService.ts @@ -13,6 +13,7 @@ import { WHISPER_MODELS, cleanTranscription } from './whisperModels'; import { audioSessionManager } from './audioSessionManager'; import { audioRecorderService } from './audioRecorderService'; import * as whisperModelFiles from './whisperModelFiles'; +import { hardwareService } from './hardware'; // Re-exported so existing consumers keep importing them from whisperService. export { WHISPER_MODELS, cleanTranscription }; @@ -94,6 +95,10 @@ export class WhisperBusyError extends Error { class WhisperService { private context: WhisperContext | null = null; private currentModelPath: string | null = null; + // Acceleration options the live context was loaded with (serialized). Used to reload + // when the user flips a toggle - a same-path load with changed options must NOT + // early-return, or the new setting silently never takes effect. + private currentLoadOpts: string = ''; private isTranscribing: boolean = false; private stopFn: (() => void) | null = null; private isReleasingContext: boolean = false; @@ -126,10 +131,21 @@ class WhisperService { return this.getModelPath(modelId).replace(/\.bin$/i, '-encoder.mlmodelc'); } - /** True when this model's CoreML encoder is present on disk (iOS only). */ + /** + * A compiled CoreML model is a DIRECTORY; a partial/interrupted extraction can leave a + * dir that exists but is broken, and whisper.cpp may crash trying to load it. So + * "present" must mean VALID, not just exists: a compiled .mlmodelc always contains + * `coremldata.bin`. Existence-only checks are the bug that lets a corrupt encoder load. + */ + private async isValidCoreMLEncoder(dir: string): Promise { + if (!(await RNFS.exists(dir))) return false; + return RNFS.exists(`${dir}/coremldata.bin`); + } + + /** True when this model's CoreML encoder is present AND valid on disk (iOS only). */ async hasCoreMLEncoder(modelId: string): Promise { if (Platform.OS !== 'ios') return false; - return RNFS.exists(this.coreMLPathFor(modelId)); + return this.isValidCoreMLEncoder(this.coreMLPathFor(modelId)); } /** @@ -144,19 +160,30 @@ class WhisperService { const model = WHISPER_MODELS.find(m => m.id === modelId); if (!model?.coreMLUrl) return false; const targetDir = this.coreMLPathFor(modelId); // ggml--encoder.mlmodelc - if (await RNFS.exists(targetDir)) return true; + if (await this.isValidCoreMLEncoder(targetDir)) return true; + // A prior run may have left a stale/partial dir that failed validation - clear it + // so a corrupt encoder never lingers and blocks a clean re-fetch. + await RNFS.unlink(targetDir).catch(() => {}); await this.ensureModelsDirExists(); const zipPath = `${this.getModelsDir()}/ggml-${modelId}-encoder.mlmodelc.zip`; - await RNFS.unlink(zipPath).catch(() => {}); // clear any partial from a prior run + // Extract into a TEMP dir first, then atomic-rename into place only after an + // integrity check. A network drop mid-download must never leave a half-extracted + // encoder that whisper.cpp then tries (and crashes) to load. + const tmpDir = `${this.getModelsDir()}/.coreml-tmp-${modelId}`; + await RNFS.unlink(zipPath).catch(() => {}); + await RNFS.unlink(tmpDir).catch(() => {}); + const STALL_MS = 30000; // no bytes for 30s => treat as a dropped connection try { logger.log(`[Whisper][CoreML] START download ${modelId} from ${model.coreMLUrl}`); const t0 = Date.now(); let lastPct = -1; - const { promise } = RNFS.downloadFile({ + let lastProgressAt = Date.now(); + const { jobId, promise } = RNFS.downloadFile({ fromUrl: model.coreMLUrl, toFile: zipPath, progressInterval: 500, progress: (r) => { + lastProgressAt = Date.now(); if (r.contentLength <= 0) return; const frac = r.bytesWritten / r.contentLength; onProgress?.(frac); @@ -167,32 +194,84 @@ class WhisperService { } }, }); - const res = await promise; + // Stall watchdog: RNFS/iOS won't reject a dropped connection quickly (it hangs on + // OS defaults), so abort ourselves if no bytes arrive for STALL_MS -> clean CPU + // fallback instead of a wedged background fetch. + let stalled = false; + const watchdog = setInterval(() => { + if (Date.now() - lastProgressAt > STALL_MS) { + stalled = true; + RNFS.stopDownload(jobId); + } + }, 5000); + let res; + try { + res = await promise; + } finally { + clearInterval(watchdog); + } + if (stalled) throw new Error('download stalled (network drop)'); if (res.statusCode && res.statusCode >= 400) throw new Error(`HTTP ${res.statusCode}`); const zipMB = (Number((await RNFS.stat(zipPath)).size) / 1e6).toFixed(0); - logger.log(`[Whisper][CoreML] downloaded ${modelId} (${zipMB} MB) in ${((Date.now() - t0) / 1000).toFixed(1)}s — unzipping`); - await unzip(zipPath, this.getModelsDir()); + logger.log(`[Whisper][CoreML] downloaded ${modelId} (${zipMB} MB) in ${((Date.now() - t0) / 1000).toFixed(1)}s — unzipping to temp`); + // unzip throws on a truncated archive; that plus the integrity check below means a + // partial download can never become the live encoder. + await unzip(zipPath, tmpDir); await RNFS.unlink(zipPath).catch(() => {}); - // The zip's top-level dir is named after the SOURCE encoder in the URL. For - // most models that already equals targetDir; when a model reuses another's - // encoder (tdrz -> small.en) the names differ, so rename it into place. + // The zip's top-level dir is named after the SOURCE encoder in the URL (a model may + // reuse another's encoder, e.g. tdrz -> small.en); fall back to the temp root if the + // archive extracted files directly. const extractedName = model.coreMLUrl.split('/').pop()!.replace(/\.zip$/i, ''); - const extractedDir = `${this.getModelsDir()}/${extractedName}`; - if (extractedDir !== targetDir && (await RNFS.exists(extractedDir))) { - await RNFS.unlink(targetDir).catch(() => {}); // clear any stale target - await RNFS.moveFile(extractedDir, targetDir); - logger.log(`[Whisper][CoreML] reused ${extractedName} → ${targetDir.split('/').pop()}`); + const extractedDir = `${tmpDir}/${extractedName}`; + const src = (await RNFS.exists(extractedDir)) ? extractedDir : tmpDir; + if (!(await this.isValidCoreMLEncoder(src))) { + throw new Error('extracted CoreML encoder failed integrity check (partial/corrupt)'); } - const ok = await RNFS.exists(targetDir); - logger.log(`[Whisper][CoreML] ${ok ? `READY for ${modelId} — next load will use the Neural Engine` : `FAILED for ${modelId}: no encoder dir after unzip`}`); + await RNFS.unlink(targetDir).catch(() => {}); // clear any stale target + await RNFS.moveFile(src, targetDir); + await RNFS.unlink(tmpDir).catch(() => {}); // remove the (now-empty) temp parent + const ok = await this.isValidCoreMLEncoder(targetDir); + const readyMsg = ok + ? `READY for ${modelId} — next load will use the Neural Engine` + : `FAILED for ${modelId}: invalid after move`; + logger.log(`[Whisper][CoreML] ${readyMsg}`); return ok; } catch (e) { logger.warn(`[Whisper][CoreML] fetch FAILED for ${modelId} (staying CPU-only): ${String(e)}`); await RNFS.unlink(zipPath).catch(() => {}); + await RNFS.unlink(tmpDir).catch(() => {}); return false; } } + /** + * iOS CoreML (Neural Engine) gate for a load. Returns whether to use CoreML plus a + * human reason for the log. TWO gates, both required: (1) the user's "Neural Engine" + * setting (a real off-switch is the only reliable escape on a device where CoreML + * crashes/garbles - a native failure we can't catch, with no denylist), and (2) a + * VALID encoder asset on disk. When enabled but missing, kick off a one-time + * background backfill so the NEXT load uses the ANE (this load stays CPU). Non-iOS + * never uses CoreML. + */ + private async resolveCoreML( + modelPath: string, + coreMLEnabled: boolean, + ): Promise<{ useCoreML: boolean; reason: string }> { + if (Platform.OS !== 'ios') return { useCoreML: false, reason: 'not iOS' }; + if (!coreMLEnabled) return { useCoreML: false, reason: 'user disabled (Neural Engine off) - forcing CPU' }; + const coreMLPath = modelPath.replace(/\.bin$/i, '-encoder.mlmodelc'); + if (await this.isValidCoreMLEncoder(coreMLPath)) { + return { useCoreML: true, reason: `encoder asset present (${coreMLPath.split('/').pop()})` }; + } + const model = WHISPER_MODELS.find(m => this.getModelPath(m.id) === modelPath); + if (model?.coreMLUrl && !this.coreMLBackfillTried.has(model.id)) { + this.coreMLBackfillTried.add(model.id); + logger.log(`[Whisper][CoreML] encoder missing for ${model.id}; fetching in background for next load`); + this.ensureCoreMLEncoder(model.id).catch(() => {}); + } + return { useCoreML: false, reason: 'encoder asset missing - CPU this load, backfilling in background' }; + } + async downloadModel(modelId: string, onProgress?: (progress: number) => void): Promise { const model = WHISPER_MODELS.find(m => m.id === modelId); if (!model) throw new Error(`Unknown model: ${modelId}`); @@ -289,6 +368,13 @@ class WhisperService { await RNFS.unlink(destPath).catch(err => logger.error('[Whisper] Failed to delete invalid model file:', err)); throw new Error(`Downloaded model file is invalid: ${validationError instanceof Error ? validationError.message : 'unknown error'}`); } + // iOS: fetch the CoreML encoder before we drop the download row, so the download + // stays "in progress" until the model is truly ANE-ready (not a silent second fetch + // after the bar disappears). Non-fatal: on any failure the model is already usable on + // CPU, and loadModel will backfill the encoder later. + if (Platform.OS === 'ios' && model.coreMLUrl) { + await this.ensureCoreMLEncoder(modelId).catch(() => {}); + } } finally { // Completed STT models are listed from disk by useVoiceDownloadItems, so // the in-flight store entry must be dropped on success AND failure: leaving @@ -296,11 +382,6 @@ class WhisperService { // refuses when an entry already exists for this modelKey). useDownloadStore.getState().remove(modelKey); } - // iOS: also fetch the CoreML encoder so the ANE can run the encoder. Non-fatal - // and no-op off-iOS / when unpublished - the model is already usable on CPU. - if (Platform.OS === 'ios') { - await this.ensureCoreMLEncoder(modelId).catch(() => {}); - } logger.log(`[Whisper] Downloaded to ${destPath}`); return destPath; } @@ -320,6 +401,12 @@ class WhisperService { } const path = this.getModelPath(modelId); if (await RNFS.exists(path)) await RNFS.unlink(path); + // Also remove the CoreML encoder (iOS ANE asset). It's a derived companion to the + // .bin - useless on its own - so deleting the model must delete it too, else it + // orphans a ~tens-of-MB .mlmodelc directory on disk. RNFS.unlink removes the dir + // recursively; no-op when absent (Android, or a model with no encoder). + const encoderPath = this.coreMLPathFor(modelId); + if (await RNFS.exists(encoderPath)) await RNFS.unlink(encoderPath); } /** @@ -357,11 +444,18 @@ class WhisperService { async loadModel( modelPath: string, - options?: { useGpu?: boolean; useFlashAttn?: boolean; useCoreML?: boolean }, + options?: { useGpu?: boolean; useCoreML?: boolean }, ): Promise { wireNativeWhisperLog(); - if (this.context && this.currentModelPath !== modelPath) await this.unloadModel(); - if (this.context && this.currentModelPath === modelPath) return; + // Reload when the model OR its acceleration options change - otherwise flipping the + // Neural Engine / GPU / Flash toggle silently wouldn't take effect while a context is + // live (loadModel used to early-return on same-path regardless of options). Keyed on + // the REQUESTED options (default coreML ON) so a user toggle change forces a reload. + const optsKey = `gpu=${options?.useGpu ?? false},coreml=${options?.useCoreML ?? true}`; + if (this.context && (this.currentModelPath !== modelPath || this.currentLoadOpts !== optsKey)) { + await this.unloadModel(); + } + if (this.context && this.currentModelPath === modelPath && this.currentLoadOpts === optsKey) return; if (this.isReleasingContext) { logger.log('[WhisperService] Waiting for context release to finish before loading'); await this.contextReleasePromise; @@ -371,43 +465,35 @@ class WhisperService { // Native initWithModelPath calls abort() on invalid files, crashing the app. await this.validateModelFile(modelPath); - // CoreML runs the whisper encoder on the Apple Neural Engine (iOS only, ~2-3x - // faster encode, frees the CPU). Drive it purely off the per-model encoder - // asset: if ggml--encoder.mlmodelc sits next to the .bin we use CoreML, - // else CPU. Enabling CoreML WITHOUT that asset makes whisper.rn fail to load it - // and can crash at transcribe (0%) on some A12 devices (e.g. iPhone XS), so - // presence is the gate - not a user toggle. Non-iOS never uses CoreML. - let useCoreML = false; - if (Platform.OS === 'ios') { - const coreMLPath = modelPath.replace(/\.bin$/i, '-encoder.mlmodelc'); - useCoreML = await RNFS.exists(coreMLPath); - if (useCoreML) { - logger.log(`[Whisper][CoreML] encoder PRESENT for this model (${coreMLPath.split('/').pop()}) — requesting Neural Engine. Watch for whisper.cpp native line "Core ML model loaded" to confirm actual use.`); - } else { - if (options?.useCoreML) logger.warn('[Whisper] CoreML requested but encoder asset missing; using CPU instead'); - // Backfill: model was downloaded before CoreML shipping. Fetch its encoder - // in the background (non-blocking, once per session) so the NEXT load uses - // the ANE. This load stays CPU. - const model = WHISPER_MODELS.find(m => this.getModelPath(m.id) === modelPath); - if (model?.coreMLUrl && !this.coreMLBackfillTried.has(model.id)) { - this.coreMLBackfillTried.add(model.id); - logger.log(`[Whisper] CoreML encoder missing for ${model.id}; fetching in background for next load`); - this.ensureCoreMLEncoder(model.id).catch(() => {}); - } - } - } - + // Resolve the CoreML (Neural Engine) gate: user setting (default ON) AND a valid + // encoder asset present. resolveCoreML also kicks off a one-time background backfill + // when enabled-but-missing. + const coreMLEnabled = options?.useCoreML ?? true; + const { useCoreML, reason: coreMLReason } = await this.resolveCoreML(modelPath, coreMLEnabled); + + // GPU offload is enforced HERE, at the whisper load site, so the stored setting can + // never request the GPU on an ineligible device regardless of the settings UI. One + // cross-platform rule via hardwareService.whisperSupportsGpu(): iOS -> Metal (real + // device, >4GB). Android has no whisper GPU backend (the ggml-OpenCL port was removed + // for crashing OpenCL-2.0 devices), so whisper runs on CPU there. Gates ONLY whisper. + const useGpu = (options?.useGpu ?? false) && (await hardwareService.whisperSupportsGpu()); + // One structured line stating the RESOLVED acceleration config for this load, so a + // log pull can confirm exactly what was requested. For the DEFINITIVE proof CoreML + // actually engaged (vs silently falling back), watch the whisper.cpp NATIVE line + // "Core ML model loaded" that wireNativeWhisperLog pipes right after this. logger.log( - `[Whisper] Loading model: ${modelPath} useGpu=${options?.useGpu ?? false} ` + - `useFlashAttn=${options?.useFlashAttn ?? false} useCoreML=${useCoreML}`, + `[Whisper][ACCEL] resolved for load: platform=${Platform.OS} ` + + `coreML=${useCoreML} (enabled=${coreMLEnabled}; ${coreMLReason}) gpu=${useGpu}`, ); try { - // useGpu/useFlashAttn/useCoreMLIos are real whisper.rn runtime options but - // absent from this version's WhisperContextOptions type, so pass via a cast. + // useGpu/useCoreMLIos are real whisper.rn runtime options but absent from this + // version's WhisperContextOptions type, so pass via a cast. Flash attention is + // intentionally forced off (removed as a setting): it only helps on the GPU, our + // encoder is on the ANE, and it's unsupported by the ggml OpenCL backend. const initOpts: Record = { filePath: modelPath, - useGpu: options?.useGpu ?? false, - useFlashAttn: options?.useFlashAttn ?? false, + useGpu, + useFlashAttn: false, useCoreMLIos: useCoreML, }; // Time initWhisper: this covers reading the .bin into memory AND, when @@ -418,11 +504,22 @@ class WhisperService { const tInit = Date.now(); this.context = await initWhisper(initOpts as unknown as Parameters[0]); this.currentModelPath = modelPath; - logger.log(`[Whisper] Model loaded successfully — initWhisper took ${((Date.now() - tInit) / 1000).toFixed(1)}s (useCoreML=${useCoreML})`); + this.currentLoadOpts = optsKey; + const ctxGpu = (this.context as unknown as { gpu?: boolean }).gpu; + // Post-load confirmation: context.gpu reflects whether the Metal backend is live; + // if coreML was requested, the ABSENCE of the native "Core ML model loaded" line + // above means it silently fell back to CPU (e.g. a corrupt/partial encoder). + const coreMLNote = useCoreML + ? ' — confirm the native "Core ML model loaded" line; its absence = CPU fallback.' + : ''; + logger.log( + `[Whisper][ACCEL] context ready in ${((Date.now() - tInit) / 1000).toFixed(1)}s — context.gpu=${ctxGpu} coreMLRequested=${useCoreML}${coreMLNote}`, + ); } catch (error) { logger.error('[Whisper] Failed to load model:', error); this.context = null; this.currentModelPath = null; + this.currentLoadOpts = ''; throw error; } } @@ -447,7 +544,7 @@ class WhisperService { this.isReleasingContext = true; this.contextReleasePromise = (async () => { try { await this.context!.release(); } catch (error) { logger.error('[WhisperService] Error releasing context:', error); } - finally { this.context = null; this.currentModelPath = null; this.isReleasingContext = false; } + finally { this.context = null; this.currentModelPath = null; this.currentLoadOpts = ''; this.isReleasingContext = false; } })() await this.contextReleasePromise; } diff --git a/src/stores/whisperStore.ts b/src/stores/whisperStore.ts index 596615a69..bdf25701b 100644 --- a/src/stores/whisperStore.ts +++ b/src/stores/whisperStore.ts @@ -42,7 +42,7 @@ interface WhisperState { downloadModel: (modelId: string) => Promise; /** Activate an already-downloaded model without re-downloading. */ selectModel: (modelId: string) => Promise; - loadModel: (options?: { useGpu?: boolean; useFlashAttn?: boolean; useCoreML?: boolean }) => Promise; + loadModel: (options?: { useGpu?: boolean; useCoreML?: boolean }) => Promise; unloadModel: () => Promise; deleteModel: () => Promise; /** Delete a specific on-disk model (active or not). */ @@ -134,7 +134,7 @@ export const useWhisperStore = create()( } }, - loadModel: async (options?: { useGpu?: boolean; useFlashAttn?: boolean; useCoreML?: boolean }): Promise => { + loadModel: async (options?: { useGpu?: boolean; useCoreML?: boolean }): Promise => { const { downloadedModelId, isModelLoading } = get(); if (!downloadedModelId) { set({ error: 'No model downloaded' });