Karaoke Captions
Karaoke captions: word-by-word highlighted subtitles synced to speech. A caption line that fills word by word as it is spoken — heavy outlined type that reads on any footage.
Source
components/kuchi/karaoke-captions.tsx
1"use client";23import { loadFont } from "@remotion/google-fonts/Montserrat";4import { loadFont as loadRoboto } from "@remotion/google-fonts/Roboto";5import { useEffect, useLayoutEffect, useRef, useState } from "react";6import {7 continueRender,8 delayRender,9 Easing,10 getRemotionEnvironment,11 interpolate,12 useCurrentFrame,13 useVideoConfig,14} from "remotion";15import {16 resolveFont,17 type KuchiTheme,18 useKuchiTheme,19 withAlpha,20} from "@/lib/kuchi-ui";2122// The face the look is made of. Inter at 600 is a subtitle; a caption is a heavy23// geometric grotesque carrying an outline.24const { fontFamily: MONTSERRAT, waitUntilDone: montserratReady } = loadFont(25 "normal",26 {27 weights: ["700", "800", "900"],28 subsets: ["latin"],29 },30);3132// The `boxed` look is the YouTube auto-caption, which is set in Roboto — a33// neutral grotesque, not a geometric one.34const { fontFamily: ROBOTO } = loadRoboto("normal", {35 weights: ["500", "700"],36 subsets: ["latin"],37});3839export type CaptionTheme = "light" | "dark";40export type SafeAreaPreset = "landscape" | "portrait" | "square";4142export interface CaptionLine {43 text: string;44 /** First frame the line is on screen and the fill sweep begins. */45 startFrame: number;46 /** Frame the sweep completes; the line fades out shortly after. */47 endFrame: number;48 /**49 * Absolute frame each word starts filling (one entry per word).50 * Omitted → words are spread evenly across [startFrame, endFrame].51 */52 wordTimings?: number[];53 /** Word indices rendered in the accent color with a subtle scale-up. */54 emphasize?: number[];55}5657export interface KaraokeCaptionsProps {58 /**59 * Word-level captions in MILLISECONDS — the shape Whisper, CapCut and Remotion's60 * own `@remotion/captions` all speak. This is the input you will actually have.61 */62 captions?: Caption[];63 /** An .srt, pasted straight in. */64 srt?: string;65 /** The look. See {@link KaraokePreset}. */66 preset?: KaraokePreset;67 /** Outline width as a fraction of the font size. 0 turns it off. */68 strokeRatio?: number;69 strokeColor?: string;70 uppercase?: boolean;71 /** Timed caption lines. Omit to build a single line from `text`. */72 lines?: CaptionLine[];73 /** Convenience single line used when `lines` is not provided. */74 text?: string;75 /** Comma-separated words of `text` to emphasize (case-insensitive). */76 emphasize?: string;77 /** Design-system token overrides. */78 theme?: Partial<KuchiTheme>;79 /**80 * The face this scene paints its words in — a label from `fonts.ts`81 * ("Inter", "Space Grotesk", "Instrument Serif") or a CSS family you have82 * loaded yourself. Unset, the scene keeps the face it was designed around.83 *84 * Overrides `theme.fontFamily`, which is how a brand kit re-skins a whole85 * timeline from one value.86 */87 fontFamily?: string;88 /** Light or dark pill chrome. */89 mode?: CaptionTheme;90 /** Color of emphasized words. */91 accentColor?: string;92 fontSize?: number;93 fontWeight?: number;94 /** Safe-area preset positioning the caption block per aspect ratio. */95 aspect?: SafeAreaPreset;96 /** Rounded surface behind the line. */97 pill?: boolean;98 /** Scale emphasized words grow to as their fill completes. */99 emphasisScale?: number;100 /** Unfilled (not-yet-spoken) word color override. Defaults per theme. */101 baseColor?: string;102 /** Filled (spoken) word color override. Defaults per theme. */103 fillColor?: string;104 speed?: number;105 className?: string;106}107108/**109 * The looks.110 *111 * - `karaoke` — a heavy outlined line; each word snaps from dimmed to full white112 * as it is spoken. The read-along look.113 * - `highlight` — the spoken word rides a rounded accent bar that sweeps along the114 * line, like a marker running under the voice.115 * - `clean` — the quiet pill. No outline. (The old default.)116 */117export type KaraokePreset = "boxed" | "karaoke" | "highlight" | "clean";118119export interface KaraokeLook {120 weight: number;121 uppercase: boolean;122 /** Font size as a fraction of the frame's SHORT side. */123 sizeRatio: number;124 strokeRatio: number;125 bar: boolean;126 pill: boolean;127 shadow: boolean;128 /** Per-line solid box behind the line (YouTube auto-caption look). */129 box?: boolean;130}131132export const KARAOKE_LOOKS: Record<KaraokePreset, KaraokeLook> = {133 /**134 * The default: the YouTube auto-caption — white Roboto on a solid black box135 * that wraps each line tight (`box-decoration-break: clone`). No outline, no136 * sweep; the box carries the contrast.137 */138 boxed: {139 weight: 700,140 uppercase: false,141 sizeRatio: 0.062,142 strokeRatio: 0,143 bar: false,144 pill: false,145 shadow: false,146 box: true,147 },148 karaoke: {149 weight: 900,150 uppercase: true,151 sizeRatio: 0.085,152 strokeRatio: 0.09,153 bar: false,154 pill: false,155 shadow: true,156 },157 highlight: {158 weight: 800,159 uppercase: false,160 sizeRatio: 0.08,161 strokeRatio: 0.07,162 bar: true,163 pill: false,164 shadow: true,165 },166 clean: {167 weight: 600,168 uppercase: false,169 sizeRatio: 0.067,170 strokeRatio: 0,171 bar: false,172 pill: true,173 shadow: false,174 },175};176177/** Frames the line takes to rise/fade in and to fade out past `endFrame`. */178export const LINE_FADE_FRAMES = 8;179180/**181 * The pill's chrome, taken from the design system.182 *183 * A caption pill IS app chrome — a card floating over footage — so it follows184 * the tokens. The burned-in type it carries does not: its accent, its outline185 * and its white fill are the caption's look, and stay props.186 */187export function karaokePalette(t: KuchiTheme): {188 base: string;189 fill: string;190 pillBg: string;191 pillBorder: string;192 pillShadow: string;193} {194 return {195 base: t.mutedForeground,196 fill: t.foreground,197 pillBg: withAlpha(t.card, 0.92),198 pillBorder: t.border,199 pillShadow: `0 8px 24px ${withAlpha(t.foreground, 0.1)}`,200 };201}202203/**204 * Aspect-ratio safe areas for social placements. Shared caption-block205 * positioning (portrait keeps clear of TikTok/Reels UI chrome).206 */207export const CAPTION_SAFE_AREAS: Record<208 SafeAreaPreset,209 { bottom: string; horizontal: string }210> = {211 landscape: { bottom: "8%", horizontal: "10%" },212 portrait: { bottom: "18%", horizontal: "7%" },213 square: { bottom: "10%", horizontal: "8%" },214};215216/** Whitespace-split words, empties dropped. */217export function splitWords(text: string): string[] {218 return text.split(/\s+/).filter((w) => w.length > 0);219}220221export interface WordWindow {222 start: number;223 end: number;224}225226/**227 * Per-word fill windows for a line. Uses `wordTimings` (absolute start228 * frames; a word ends when the next begins, the last at `endFrame`) when229 * provided, otherwise splits [startFrame, endFrame] evenly.230 */231export function wordWindows(line: CaptionLine): WordWindow[] {232 const words = splitWords(line.text);233 const n = words.length;234 if (n === 0) return [];235 const { startFrame, endFrame, wordTimings } = line;236 if (wordTimings && wordTimings.length > 0) {237 return words.map((_, i) => {238 const start = wordTimings[i] ?? wordTimings[wordTimings.length - 1];239 const end = wordTimings[i + 1] ?? endFrame;240 return { start, end: Math.max(end, start + 1) };241 });242 }243 const step = (endFrame - startFrame) / n;244 return words.map((_, i) => ({245 start: startFrame + step * i,246 end: startFrame + step * (i + 1),247 }));248}249250/** 0..1 sweep progress of one word at `frame`. */251export function fillProgress(frame: number, window: WordWindow): number {252 if (window.end <= window.start) return frame >= window.end ? 1 : 0;253 return Math.min(254 1,255 Math.max(0, (frame - window.start) / (window.end - window.start)),256 );257}258259/**260 * Word indices of `text` matching the comma-separated `emphasize` list.261 * Comparison is case-insensitive and ignores surrounding punctuation.262 */263export function emphasisIndices(text: string, emphasize: string): number[] {264 const targets = emphasize265 .split(",")266 .map((w) => w.trim().toLowerCase())267 .filter((w) => w.length > 0);268 if (targets.length === 0) return [];269 const strip = (w: string) =>270 w.toLowerCase().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");271 return splitWords(text)272 .map((word, i) => (targets.indexOf(strip(word)) !== -1 ? i : -1))273 .filter((i) => i >= 0);274}275276/** Single demo line built from the convenience `text` prop. */277export function linesFromText(278 text: string,279 emphasize: string,280 startFrame: number,281 endFrame: number,282): CaptionLine[] {283 return [284 {285 text,286 startFrame,287 endFrame: Math.max(endFrame, startFrame + 1),288 emphasize: emphasisIndices(text, emphasize),289 },290 ];291}292293/* ---------------------------------------------------------------------------294 * Real transcripts.295 *296 * Nobody has frame numbers. A caption comes out of Whisper, whisper.cpp, CapCut or297 * Descript in MILLISECONDS, or as an .srt. Remotion's own `@remotion/captions` type298 * is `{ text, startMs, endMs }` — so that is what this takes.299 *300 * (Deliberately duplicated from word-captions rather than imported: this is a301 * copy-paste registry, and installing a caption line should not drag a second302 * component in with it.)303 * ------------------------------------------------------------------------- */304305/** The standard caption token. Structurally identical to `@remotion/captions`. */306export interface Caption {307 text: string;308 startMs: number;309 endMs: number;310}311312/** `00:00:01,234` (or `.234`) → 1234. Null when it isn't a timestamp. */313export function srtTimeToMs(stamp: string): number | null {314 const m = stamp.trim().match(/^(?:(\d+):)?(\d+):(\d+)[,.](\d{1,3})$/);315 if (!m) return null;316 const [, h, min, sec, frac] = m;317 return (318 (h ? Number(h) : 0) * 3600000 +319 Number(min) * 60000 +320 Number(sec) * 1000 +321 // `padEnd(3, "0")` without the ES2017 lib a stock Remotion project lacks.322 Number(frac + "000".slice(frac.length))323 );324}325326/** Cues from an .srt. Handles `,` and `.` separators and multi-line cues. */327export function parseSrt(srt: string): Caption[] {328 const out: Caption[] = [];329 for (const raw of srt330 .replace(/\r/g, "")331 .trim()332 .split(/\n\s*\n/)) {333 const lines = raw.split("\n").filter((l) => l.trim().length > 0);334 const timeLine = lines.find((l) => l.includes("-->"));335 if (!timeLine) continue;336 const [from, to] = timeLine.split("-->").map((t) => t.trim());337 const startMs = srtTimeToMs(from);338 const endMs = srtTimeToMs(to);339 if (startMs === null || endMs === null) continue;340 const text = lines341 .filter((l) => l !== timeLine && !/^\d+$/.test(l.trim()))342 .join(" ")343 .trim();344 if (text) out.push({ text, startMs, endMs });345 }346 return out;347}348349/**350 * Captions (ms) → the line model this component draws. One cue is one line; its351 * words are spread across the cue so the fill has something to track. Word-level352 * Whisper output collapses naturally: consecutive words inside `groupWithinMs` of353 * each other become one line, which is how a caption tool builds a line too.354 */355export function linesFromCaptions(356 captions: Caption[],357 fps: number,358 groupWithinMs = 1200,359): CaptionLine[] {360 if (captions.length === 0) return [];361 const toFrame = (ms: number) => (ms / 1000) * fps;362363 const groups: Caption[][] = [];364 let current: Caption[] = [];365 for (const c of captions) {366 const prev = current[current.length - 1];367 if (prev && c.startMs - prev.endMs > groupWithinMs) {368 groups.push(current);369 current = [];370 }371 current.push(c);372 }373 if (current.length) groups.push(current);374375 return groups.map((group) => {376 const words = ([] as string[]).concat(377 ...group.map((c) => c.text.split(/\s+/).filter(Boolean)),378 );379 const startMs = group[0].startMs;380 const endMs = group[group.length - 1].endMs;381 // One timing per word: a multi-word cue is spread across its own span.382 const timings: number[] = [];383 for (const c of group) {384 const parts = c.text.split(/\s+/).filter(Boolean);385 const per = Math.max(1, c.endMs - c.startMs) / Math.max(1, parts.length);386 for (let i = 0; i < parts.length; i++) {387 timings.push(toFrame(c.startMs + i * per));388 }389 }390 return {391 text: words.join(" "),392 startFrame: toFrame(startMs),393 endFrame: Math.max(toFrame(endMs), toFrame(startMs) + 1),394 wordTimings: timings,395 };396 });397}398399/**400 * Index of the line on screen at `frame` (the last one whose window contains it,401 * including its fade-out tail), or -1 when nothing is showing.402 */403export function activeLineIndex(404 lines: CaptionLine[],405 frame: number,406 fadeOut = LINE_FADE_FRAMES,407): number {408 let active = -1;409 for (let i = 0; i < lines.length; i++) {410 if (frame >= lines[i].startFrame && frame < lines[i].endFrame + fadeOut) {411 active = i;412 }413 }414 return active;415}416417/**418 * A caption line that fills word by word as it is spoken — the read-along look.419 *420 * Same three things that separate a caption from a subtitle:421 *422 * 1. **An OUTSIDE outline.** `-webkit-text-stroke` centres the stroke on the glyph423 * and eats it from the inside (measured: a 14px stroke takes a 38px stem down to424 * 22px). `paint-order: stroke fill` puts the fill back over it. Without this a425 * caption is unreadable the moment the footage behind it is bright.426 * 2. **Weight and size.** Montserrat 800–900 at ~8.5% of the frame's short side.427 * 3. **The spoken word actually lands** — it snaps to full white (or rides an accent428 * bar), rather than easing politely.429 *430 * The emphasis scale pivots on the MEASURED baseline: a browser gives glyph origins431 * no vertical sub-pixel precision, so a scale that moves the baseline makes the word432 * climb the pixel grid in whole-pixel jumps.433 */434export function KaraokeCaptions({435 captions,436 srt,437 lines,438 text = "Acme reconciles every transaction automatically",439 emphasize = "automatically",440 preset = "boxed",441 theme,442 fontFamily,443 mode = "dark",444 accentColor = "#FFE81F",445 fontSize,446 fontWeight,447 aspect = "landscape",448 pill,449 emphasisScale = 1.08,450 baseColor,451 fillColor,452 strokeColor = "#000000",453 strokeRatio,454 uppercase,455 speed = 1,456 className,457}: KaraokeCaptionsProps) {458 const frame = useCurrentFrame() * speed;459 const { durationInFrames, fps, width, height } = useVideoConfig();460461 const look = KARAOKE_LOOKS[preset] ?? KARAOKE_LOOKS.karaoke;462 const t = useKuchiTheme(theme, mode);463 const face = resolveFont(fontFamily ?? t.fontFamily) ?? MONTSERRAT;464 const palette = karaokePalette(t);465 const safeArea = CAPTION_SAFE_AREAS[aspect] ?? CAPTION_SAFE_AREAS.landscape;466467 const shortSide = Math.min(width, height);468 // 0 (or omitted) means "the preset decides" — the preset IS the design.469 const size =470 fontSize && fontSize > 0471 ? fontSize472 : Math.round(look.sizeRatio * shortSide);473 const weight = fontWeight && fontWeight > 0 ? fontWeight : look.weight;474 const stroke =475 (strokeRatio !== undefined && strokeRatio >= 0476 ? strokeRatio477 : look.strokeRatio) * size;478 const caps = uppercase ?? look.uppercase;479 const showPill = pill ?? look.pill;480481 // An outlined caption is white-on-black by definition; the muted grey of the old482 // "unspoken" colour disappears the moment there is footage behind it. Unspoken483 // words are DIMMED WHITE, which still reads.484 const unfilled =485 baseColor ?? (look.pill ? palette.base : "rgba(255,255,255,0.45)");486 const filled = fillColor ?? (look.pill ? palette.fill : "#FFFFFF");487488 // Real captions first; the evenly-paced string is only a demo.489 const resolvedLines = captions?.length490 ? linesFromCaptions(captions, fps)491 : srt && srt.trim().length > 0492 ? linesFromCaptions(parseSrt(srt), fps)493 : lines && lines.length > 0494 ? lines495 : linesFromText(496 text,497 emphasize,498 10,499 Math.max(11, durationInFrames - 24),500 );501502 const lineIndex = activeLineIndex(resolvedLines, frame);503504 // The baseline inside a word span, measured once — the pivot for the emphasis505 // scale. Guessing it from a line-height ratio is what makes an emphasised word506 // judder.507 const probeRef = useRef<HTMLSpanElement>(null);508 const baseRef = useRef<HTMLSpanElement>(null);509 const [baselineY, setBaselineY] = useState<number | null>(null);510 // The delayRender exists only to hold the *mp4 render* on its first frame until511 // the baseline is measured — there the font is already loaded, so that one512 // measurement is exact. In the Player we never block: `baselineY === null`513 // falls back to a line-height pivot until the mount measurement lands, which is514 // invisible on a looping preview. Blocking the Player here is what stranded515 // every caption card behind an orphaned delayRender handle that React516 // StrictMode's double-invoked initializer creates and never clears.517 const [handle] = useState(() =>518 getRemotionEnvironment().isRendering519 ? delayRender(520 "karaoke-captions: measuring the baseline for the emphasis pivot",521 )522 : null,523 );524525 useLayoutEffect(() => {526 const probe = probeRef.current;527 const base = baseRef.current;528 if (probe && base) {529 setBaselineY(530 base.getBoundingClientRect().top - probe.getBoundingClientRect().top,531 );532 } else if (handle != null) {533 // Nothing to measure in a render — release it rather than hang the frame.534 continueRender(handle);535 }536 // Re-measure once Montserrat lands: a best-effort refinement for the Player537 // (the render already has the font). Never blocks — the pivot above is set.538 let cancelled = false;539 void montserratReady().then(() => {540 if (cancelled) return;541 const p = probeRef.current;542 const b = baseRef.current;543 if (p && b) {544 setBaselineY(545 b.getBoundingClientRect().top - p.getBoundingClientRect().top,546 );547 }548 });549 return () => {550 cancelled = true;551 };552 }, [handle]);553554 // In a render, release the held first frame the moment the baseline is known.555 useEffect(() => {556 if (baselineY != null && handle != null) continueRender(handle);557 }, [baselineY, handle]);558559 const lineHeight = Math.round(size * 1.2);560 const willChange = getRemotionEnvironment().isRendering561 ? undefined562 : ("transform" as const);563564 const typeStyle = {565 fontFamily: face,566 fontWeight: weight,567 fontSize: size,568 lineHeight: `${lineHeight}px`,569 letterSpacing: caps ? "-0.005em" : "-0.015em",570 textRendering: "geometricPrecision" as const,571 fontVariantLigatures: "none" as const,572 };573574 const outline =575 stroke > 0576 ? {577 WebkitTextStrokeWidth: `${stroke}px`,578 WebkitTextStrokeColor: strokeColor,579 // Stroke first, fill over it — an OUTSIDE outline, not one eating the glyph.580 paintOrder: "stroke fill" as const,581 }582 : {};583584 const shadow = look.shadow585 ? { textShadow: `0 ${size * 0.05}px ${size * 0.045}px rgba(0,0,0,0.42)` }586 : {};587588 const probe = (589 <span590 ref={probeRef}591 aria-hidden592 style={{593 ...typeStyle,594 position: "absolute",595 left: -99999,596 top: 0,597 visibility: "hidden",598 display: "inline-block",599 }}600 >601 Hg602 <span603 ref={baseRef}604 style={{ display: "inline-block", width: 0, height: 0 }}605 />606 </span>607 );608609 if (lineIndex === -1) {610 // The probe still has to mount, or delayRender never resolves and the render hangs.611 return (612 <div style={{ position: "absolute", inset: 0, pointerEvents: "none" }}>613 {probe}614 </div>615 );616 }617618 const line = resolvedLines[lineIndex];619 const words = splitWords(line.text);620 const windows = wordWindows(line);621 const emphasized = new Set(line.emphasize ?? []);622623 const enter = interpolate(624 frame - line.startFrame,625 [0, LINE_FADE_FRAMES],626 [0, 1],627 { extrapolateLeft: "clamp", extrapolateRight: "clamp" },628 );629 const exit = interpolate(630 frame - line.endFrame,631 [0, LINE_FADE_FRAMES],632 [1, 0],633 { extrapolateLeft: "clamp", extrapolateRight: "clamp" },634 );635636 return (637 <div638 className={className}639 style={{640 position: "absolute",641 inset: 0,642 display: "flex",643 alignItems: "flex-end",644 justifyContent: "center",645 paddingBottom: safeArea.bottom,646 paddingLeft: safeArea.horizontal,647 paddingRight: safeArea.horizontal,648 pointerEvents: "none",649 }}650 >651 {probe}652 {look.box ? (653 <div654 style={{655 maxWidth: Math.round(width * 0.82),656 textAlign: "center",657 opacity: Math.min(enter, exit),658 }}659 >660 {/* White Roboto on a solid black box that clones per line — the661 YouTube auto-caption look. Inline text so the box wraps each line. */}662 <span663 style={{664 ...typeStyle,665 fontFamily: ROBOTO,666 display: "inline",667 color: "#FFFFFF",668 backgroundColor: "rgba(0,0,0,0.9)",669 padding: `${size * 0.06}px ${size * 0.32}px`,670 boxDecorationBreak: "clone",671 WebkitBoxDecorationBreak: "clone",672 lineHeight: `${Math.round(size * 1.46)}px`,673 borderRadius: 2,674 }}675 >676 {caps ? line.text.toUpperCase() : line.text}677 </span>678 </div>679 ) : (680 <div681 style={{682 display: "flex",683 flexWrap: "wrap",684 alignItems: "flex-end",685 justifyContent: "center",686 columnGap: size * 0.28,687 rowGap: size * 0.06,688 textAlign: "center",689 opacity: Math.min(enter, exit),690 ...(showPill691 ? {692 padding: `${size * 0.3}px ${size * 0.55}px`,693 borderRadius: size * 0.24,694 background: palette.pillBg,695 border: `1px solid ${palette.pillBorder}`,696 boxShadow: palette.pillShadow,697 }698 : {}),699 }}700 >701 {words.map((word, i) => {702 const progress = fillProgress(frame, windows[i]);703 const spoken = progress >= 0.999;704 const isEmphasized = emphasized.has(i);705 const scale = isEmphasized706 ? interpolate(progress, [0, 1], [1, emphasisScale], {707 easing: Easing.out(Easing.cubic),708 extrapolateLeft: "clamp",709 extrapolateRight: "clamp",710 })711 : 1;712713 // The bar preset: the spoken word rides a rounded accent block.714 const onBar =715 look.bar && progress > 0 && !spoken ? false : look.bar && spoken;716 const color = look.bar717 ? spoken718 ? "#0B0B0C"719 : unfilled720 : isEmphasized && spoken721 ? accentColor722 : spoken723 ? filled724 : unfilled;725726 return (727 <span728 // biome-ignore lint/suspicious/noArrayIndexKey: words in a line repeat, so the index is what makes each key unique729 key={`${word}-${i}`}730 style={{731 ...typeStyle,732 ...outline,733 ...shadow,734 display: "inline-block",735 color,736 transform: `scale(${scale})`,737 // Pivot on the BASELINE. `50% 80%` is a guess, and a guess drags the738 // word down the pixel grid as it scales.739 transformOrigin:740 baselineY === null ? "50% 82%" : `50% ${baselineY}px`,741 willChange,742 ...(onBar743 ? {744 background: accentColor,745 borderRadius: size * 0.14,746 padding: `0 ${size * 0.11}px`,747 WebkitTextStrokeWidth: 0,748 }749 : {}),750 }}751 >752 {caps ? word.toUpperCase() : word}753 </span>754 );755 })}756 </div>757 )}758 </div>759 );760}


