Skip to content

feat(captions): generate auto-captions as styled subtitles #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joaothaira merged 3 commits into from
Jun 11, 2026
Merged

feat(captions): generate auto-captions as styled subtitles #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6e1f872
feat(captions): generate auto-captions as styled subtitles
ThairaHub Jun 11, 2026
ac4e585
feat(timeline): own lane for styled subtitles
ThairaHub Jun 11, 2026
f447ce1
feat(timeline): clicking a subtitle opens its editor in the sidebar
ThairaHub Jun 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions src/components/video-editor/EnterpriseSettingsPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,8 @@ interface EnterpriseSettingsPanelProps {
subtitleStyle?: import("./types").SubtitleStyle;
onSubtitleStyleChange?: (style: Partial<import("./types").SubtitleStyle>) => void;
onSubtitleTextChange?: (id: string, text: string) => void;
/** Subtitle selected on the timeline; opens the Subtitles section and scrolls to it. */
selectedSubtitleId?: string | null;
onGenerateSubtitles?: () => void;
isGeneratingSubtitles?: boolean;
onClearSubtitles?: () => void;
Expand Down Expand Up @@ -284,6 +286,7 @@ export function EnterpriseSettingsPanel({
subtitleStyle,
onSubtitleStyleChange,
onSubtitleTextChange,
selectedSubtitleId,
onGenerateSubtitles,
isGeneratingSubtitles = false,
onClearSubtitles,
Expand All @@ -292,6 +295,25 @@ export function EnterpriseSettingsPanel({
const [wallpaperPaths, setWallpaperPaths] = useState<string[]>([]);
const [customImages, setCustomImages] = useState<string[]>([]);
const fileInputRef = useRef<HTMLInputElement>(null);
const [openSections, setOpenSections] = useState<string[]>(() =>
hasWebcam ? ["layout", "effects", "background"] : ["effects", "background"],
);
const subtitleItemRefs = useRef(new Map<string, HTMLTextAreaElement>());

// Timeline selection drives the sidebar: open the Subtitles section and bring
// the selected line's editor into view (after the accordion expand settles).
useEffect(() => {
if (!selectedSubtitleId) return;
setOpenSections((prev) => (prev.includes("subtitles") ? prev : [...prev, "subtitles"]));
const timer = window.setTimeout(() => {
const el = subtitleItemRefs.current.get(selectedSubtitleId);
if (el) {
el.scrollIntoView({ behavior: "smooth", block: "center" });
el.focus({ preventScroll: true });
}
}, 220);
return () => window.clearTimeout(timer);
}, [selectedSubtitleId]);

useEffect(() => {
let mounted = true;
Expand Down Expand Up @@ -716,7 +738,8 @@ export function EnterpriseSettingsPanel({

<Accordion
type="multiple"
defaultValue={hasWebcam ? ["layout", "effects", "background"] : ["effects", "background"]}
value={openSections}
onValueChange={setOpenSections}
className="space-y-1"
>
<AccordionItem value="layout" className="border-white/5 rounded-xl bg-white/[0.02] px-3">
Expand Down Expand Up @@ -1531,13 +1554,22 @@ export function EnterpriseSettingsPanel({
const mm = Math.floor(startSec / 60);
const ss = String(Math.floor(startSec % 60)).padStart(2, "0");
const ts = `${mm}:${ss}`;
const isSelected = sub.id === selectedSubtitleId;
return (
<div
key={sub.id}
className="group rounded-lg bg-white/[0.03] border border-white/5 p-2"
className={`group rounded-lg bg-white/[0.03] border p-2 ${
isSelected
? "border-[#14b8a6]/60 bg-[#14b8a6]/[0.06]"
: "border-white/5"
}`}
>
<div className="text-[10px] text-slate-600 mb-1 font-mono">{ts}</div>
<textarea
ref={(el) => {
if (el) subtitleItemRefs.current.set(sub.id, el);
else subtitleItemRefs.current.delete(sub.id);
}}
value={sub.text}
onChange={(e) => onSubtitleTextChange?.(sub.id, e.target.value)}
rows={2}
Expand Down
153 changes: 127 additions & 26 deletions src/components/video-editor/VideoEditor.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import { type Locale } from "@/i18n/config";
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
import {
captionSegmentsToAnnotationRegions,
captionSegmentsToSubtitleItems,
extractMono16kFromVideoUrl,
MAX_CAPTION_AUDIO_SEC,
reconcileAutoCaptionTimelineGaps,
Expand Down Expand Up @@ -305,6 +306,8 @@ export default function VideoEditor() {
const [selectedSpeedId, setSelectedSpeedId] = useState<string | null>(null);
const [selectedAnnotationId, setSelectedAnnotationId] = useState<string | null>(null);
const [selectedBlurId, setSelectedBlurId] = useState<string | null>(null);
const [selectedSubtitleId, setSelectedSubtitleId] = useState<string | null>(null);
const [sidebarTab, setSidebarTab] = useState("settings");
const [selectedWebcamFocusId, setSelectedWebcamFocusId] = useState<string | null>(null);
const [selectedWebcamSegmentId, setSelectedWebcamSegmentId] = useState<string | null>(null);
const [isExporting, setIsExporting] = useState(false);
Expand Down Expand Up @@ -393,6 +396,7 @@ export default function VideoEditor() {
const showCursorSettings = hasEditableCursorRecording;
const { locale, setLocale, t: rawT } = useI18n();
const [captionLanguage, setCaptionLanguage] = useState(() => captionLanguageForLocale(locale));
const [captionOutput, setCaptionOutput] = useState<"subtitles" | "annotations">("subtitles");
const t = useScopedT("editor");
const ts = useScopedT("settings");
const availableLocales = getAvailableLocales();
Expand Down Expand Up @@ -1149,6 +1153,7 @@ export default function VideoEditor() {
setSelectedSpeedId(null);
setSelectedAnnotationId(null);
setSelectedBlurId(null);
setSelectedSubtitleId(null);
}
}, []);

Expand All @@ -1159,6 +1164,7 @@ export default function VideoEditor() {
setSelectedSpeedId(null);
setSelectedAnnotationId(null);
setSelectedBlurId(null);
setSelectedSubtitleId(null);
}
}, []);

Expand All @@ -1169,6 +1175,7 @@ export default function VideoEditor() {
setSelectedTrimId(null);
setSelectedSpeedId(null);
setSelectedBlurId(null);
setSelectedSubtitleId(null);
}
}, []);

Expand All @@ -1179,6 +1186,20 @@ export default function VideoEditor() {
setSelectedTrimId(null);
setSelectedAnnotationId(null);
setSelectedSpeedId(null);
setSelectedSubtitleId(null);
}
}, []);

const handleSelectSubtitle = useCallback((id: string | null) => {
setSelectedSubtitleId(id);
if (id) {
setSelectedZoomId(null);
setSelectedTrimId(null);
setSelectedSpeedId(null);
setSelectedAnnotationId(null);
setSelectedBlurId(null);
// Jump the sidebar to this subtitle's editor (Enterprise tab).
setSidebarTab("enterprise");
}
}, []);

Expand Down Expand Up @@ -1747,6 +1768,29 @@ export default function VideoEditor() {
[selectedAnnotationId, selectedBlurId, pushState],
);

const handleSubtitleSpanChange = useCallback(
(id: string, span: Span) => {
pushState((prev) => ({
subtitleRegions: prev.subtitleRegions.map((s) =>
s.id === id ? { ...s, startMs: Math.round(span.start), endMs: Math.round(span.end) } : s,
),
}));
},
[pushState],
);

const handleSubtitleDelete = useCallback(
(id: string) => {
pushState((prev) => ({
subtitleRegions: prev.subtitleRegions.filter((s) => s.id !== id),
}));
if (selectedSubtitleId === id) {
setSelectedSubtitleId(null);
}
},
[selectedSubtitleId, pushState],
);

const handleAnnotationContentChange = useCallback(
(id: string, content: string) => {
pushState((prev) => ({
Expand Down Expand Up @@ -2093,6 +2137,12 @@ export default function VideoEditor() {
}
}, [selectedWebcamSegmentId, webcamSegments]);

useEffect(() => {
if (selectedSubtitleId && !subtitleRegions.some((s) => s.id === selectedSubtitleId)) {
setSelectedSubtitleId(null);
}
}, [selectedSubtitleId, subtitleRegions]);

const handleGenerateSubtitles = useCallback(async () => {
if (!videoSourcePath) {
toast.error("No video loaded");
Expand Down Expand Up @@ -2688,48 +2738,74 @@ export default function VideoEditor() {
}))
: segmentsRaw;

let { regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions(
segments,
nextAnnotationIdRef.current,
nextAnnotationZIndexRef.current,
{
let createdCount = 0;
if (captionOutput === "subtitles") {
let items = captionSegmentsToSubtitleItems(segments, {
minWordsPerCaption: minW,
maxWordsPerCaption: maxW,
timestampGranularity: granularity,
},
);

if (regions.length === 0 && segments.length > 0) {
({ regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions(
});
if (items.length === 0 && segments.length > 0) {
items = captionSegmentsToSubtitleItems(segments, {
minWordsPerCaption: 1,
maxWordsPerCaption: Number.MAX_SAFE_INTEGER,
timestampGranularity: granularity,
});
}
if (items.length === 0) {
toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID);
toast.info(t("autoCaptions.noneHeard"));
return;
}
// A generation describes the whole video: replace, like the sidebar
// subtitle generator does, instead of stacking duplicate lines.
pushState({ subtitleRegions: items, showSubtitles: true });
createdCount = items.length;
} else {
let { regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions(
segments,
nextAnnotationIdRef.current,
nextAnnotationZIndexRef.current,
{
minWordsPerCaption: 1,
maxWordsPerCaption: Number.MAX_SAFE_INTEGER,
minWordsPerCaption: minW,
maxWordsPerCaption: maxW,
timestampGranularity: granularity,
},
));
}
);

if (regions.length === 0) {
toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID);
toast.info(t("autoCaptions.noneHeard"));
return;
}
if (regions.length === 0 && segments.length > 0) {
({ regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions(
segments,
nextAnnotationIdRef.current,
nextAnnotationZIndexRef.current,
{
minWordsPerCaption: 1,
maxWordsPerCaption: Number.MAX_SAFE_INTEGER,
timestampGranularity: granularity,
},
));
}

if (regions.length === 0) {
toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID);
toast.info(t("autoCaptions.noneHeard"));
return;
}

pushState((prev) => ({ annotationRegions: [...prev.annotationRegions, ...regions] }));
nextAnnotationIdRef.current = nextNumericId;
nextAnnotationZIndexRef.current = nextZIndex;
pushState((prev) => ({ annotationRegions: [...prev.annotationRegions, ...regions] }));
nextAnnotationIdRef.current = nextNumericId;
nextAnnotationZIndexRef.current = nextZIndex;
createdCount = regions.length;
}

toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID);
const minutesTrunc = String(Math.round(MAX_CAPTION_AUDIO_SEC / 60));
if (truncated) {
toast.success(t("autoCaptions.done", { count: String(regions.length) }), {
toast.success(t("autoCaptions.done", { count: String(createdCount) }), {
description: t("autoCaptions.truncated", { minutes: minutesTrunc }),
});
} else {
toast.success(t("autoCaptions.done", { count: String(regions.length) }));
toast.success(t("autoCaptions.done", { count: String(createdCount) }));
}
} catch (e) {
console.error(e);
Expand All @@ -2741,7 +2817,7 @@ export default function VideoEditor() {
setIsAutoCaptioning(false);
}
},
[videoPath, trimRegions, pushState, t, captionLanguage],
[videoPath, trimRegions, pushState, t, captionLanguage, captionOutput],
);

const handleSaveDiagnostic = useCallback(async () => {
Expand Down Expand Up @@ -2821,6 +2897,21 @@ export default function VideoEditor() {
<DialogDescription>{t("autoCaptions.dialogDescription")}</DialogDescription>
</DialogHeader>
<div className="grid gap-4 py-2">
<div className="grid gap-2">
<Label htmlFor="caption-output">{t("autoCaptions.output")}</Label>
<Select
value={captionOutput}
onValueChange={(v) => setCaptionOutput(v as "subtitles" | "annotations")}
>
<SelectTrigger id="caption-output" className="h-9">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="subtitles">{t("autoCaptions.outputSubtitles")}</SelectItem>
<SelectItem value="annotations">{t("autoCaptions.outputAnnotations")}</SelectItem>
</SelectContent>
</Select>
</div>
<div className="grid gap-2">
<Label htmlFor="caption-language">{t("autoCaptions.language")}</Label>
<Select value={captionLanguage} onValueChange={setCaptionLanguage}>
Expand Down Expand Up @@ -3097,7 +3188,11 @@ export default function VideoEditor() {
</div>

<div className="editor-settings-rail min-w-0 h-full">
<Tabs defaultValue="settings" className="flex h-full flex-col">
<Tabs
value={sidebarTab}
onValueChange={setSidebarTab}
className="flex h-full flex-col"
>
<TabsList className="mx-3 mt-2 grid grid-cols-2">
<TabsTrigger value="settings">Settings</TabsTrigger>
<TabsTrigger value="enterprise">Enterprise</TabsTrigger>
Expand Down Expand Up @@ -3417,6 +3512,7 @@ export default function VideoEditor() {
subtitleStyle={subtitleStyle}
onSubtitleStyleChange={handleSubtitleStyleChange}
onSubtitleTextChange={handleSubtitleTextChange}
selectedSubtitleId={selectedSubtitleId}
onGenerateSubtitles={handleGenerateSubtitles}
isGeneratingSubtitles={isGeneratingSubtitles}
onClearSubtitles={handleClearSubtitles}
Expand Down Expand Up @@ -3478,6 +3574,11 @@ export default function VideoEditor() {
onAnnotationDelete={handleAnnotationDelete}
selectedAnnotationId={selectedAnnotationId}
onSelectAnnotation={handleSelectAnnotation}
subtitleRegions={subtitleRegions}
onSubtitleSpanChange={handleSubtitleSpanChange}
onSubtitleDelete={handleSubtitleDelete}
selectedSubtitleId={selectedSubtitleId}
onSelectSubtitle={handleSelectSubtitle}
blurRegions={blurRegions}
onBlurAdded={handleBlurAdded}
onBlurSpanChange={handleAnnotationSpanChange}
Expand Down
19 changes: 16 additions & 3 deletions src/components/video-editor/timeline/Item.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,15 @@ interface ItemProps {
zoomCustomScale?: number;
speedValue?: number;
isAutoFocus?: boolean;
variant?: "zoom" | "trim" | "annotation" | "speed" | "blur" | "webcam-focus" | "webcam-segment";
variant?:
| "zoom"
| "trim"
| "annotation"
| "speed"
| "blur"
| "webcam-focus"
| "webcam-segment"
| "subtitle";
}

// Map zoom depth to multiplier labels
Expand Down Expand Up @@ -65,6 +73,7 @@ export default function Item({
const isSpeed = variant === "speed";
const isWebcamFocus = variant === "webcam-focus";
const isWebcamSegment = variant === "webcam-segment";
const isSubtitle = variant === "subtitle";

const glassClass = isZoom
? glassStyles.glassGreen
Expand All @@ -76,7 +85,9 @@ export default function Item({
? glassStyles.glassBlue
: isWebcamSegment
? glassStyles.glassPurple
: glassStyles.glassYellow;
: isSubtitle
? glassStyles.glassTeal
: glassStyles.glassYellow;

const endCapColor = isZoom
? "#21916A"
Expand All @@ -88,7 +99,9 @@ export default function Item({
? "#4f46e5"
: isWebcamSegment
? "#9333ea"
: "#B4A046";
: isSubtitle
? "#0d9488"
: "#B4A046";

const timeLabel = useMemo(
() => `${formatMs(span.start)} – ${formatMs(span.end)}`,
Expand Down
Loading