reszta poprawek
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
---
|
||||
name: remotion-captions
|
||||
description: Transcribing, displaying and animating captions
|
||||
metadata:
|
||||
tags: subtitles, captions, remotion, json
|
||||
---
|
||||
|
||||
All captions must be processed in JSON. The captions must use the [`Caption`](https://www.remotion.dev/docs/captions/caption.md) type which is the following:
|
||||
|
||||
```ts
|
||||
import type { Caption } from "@remotion/captions";
|
||||
```
|
||||
|
||||
This is the definition:
|
||||
|
||||
```ts
|
||||
type Caption = {
|
||||
text: string;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
timestampMs: number | null;
|
||||
confidence: number | null;
|
||||
};
|
||||
```
|
||||
|
||||
## Generating captions
|
||||
|
||||
To transcribe video and audio files to generate captions, load the [transcribe-captions.md](transcribe-captions.md) file for more instructions.
|
||||
|
||||
## Displaying captions
|
||||
|
||||
To display captions in your video, load the [display-captions.md](display-captions.md) file for more instructions.
|
||||
|
||||
## Importing captions
|
||||
|
||||
To import captions from a .srt file, load the [import-srt-captions.md](import-srt-captions.md) file for more instructions.
|
||||
@@ -0,0 +1,184 @@
|
||||
---
|
||||
name: display-captions
|
||||
description: Displaying captions in Remotion with TikTok-style pages and word highlighting
|
||||
metadata:
|
||||
tags: captions, subtitles, display, tiktok, highlight
|
||||
---
|
||||
|
||||
# Displaying captions in Remotion
|
||||
|
||||
This guide explains how to display captions in Remotion, assuming you already have captions in the [`Caption`](https://www.remotion.dev/docs/captions/caption) format.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Read [Transcribing audio](transcribe-captions.md) for how to generate captions.
|
||||
|
||||
First, the [`@remotion/captions`](https://www.remotion.dev/docs/captions) package needs to be installed.
|
||||
If it is not installed, use the following command:
|
||||
|
||||
```bash
|
||||
npx remotion add @remotion/captions
|
||||
```
|
||||
|
||||
## Fetching captions
|
||||
|
||||
First, fetch your captions JSON file. Use [`useDelayRender()`](https://www.remotion.dev/docs/use-delay-render) to hold the render until the captions are loaded:
|
||||
|
||||
```tsx
|
||||
import { useState, useEffect, useCallback } from "react";
|
||||
import { AbsoluteFill, staticFile, useDelayRender } from "remotion";
|
||||
import type { Caption } from "@remotion/captions";
|
||||
|
||||
export const MyComponent: React.FC = () => {
|
||||
const [captions, setCaptions] = useState<Caption[] | null>(null);
|
||||
const { delayRender, continueRender, cancelRender } = useDelayRender();
|
||||
const [handle] = useState(() => delayRender());
|
||||
|
||||
const fetchCaptions = useCallback(async () => {
|
||||
try {
|
||||
// Assuming captions.json is in the public/ folder.
|
||||
const response = await fetch(staticFile("captions123.json"));
|
||||
const data = await response.json();
|
||||
setCaptions(data);
|
||||
continueRender(handle);
|
||||
} catch (e) {
|
||||
cancelRender(e);
|
||||
}
|
||||
}, [continueRender, cancelRender, handle]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchCaptions();
|
||||
}, [fetchCaptions]);
|
||||
|
||||
if (!captions) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return <AbsoluteFill>{/* Render captions here */}</AbsoluteFill>;
|
||||
};
|
||||
```
|
||||
|
||||
## Creating pages
|
||||
|
||||
Use `createTikTokStyleCaptions()` to group captions into pages. The `combineTokensWithinMilliseconds` option controls how many words appear at once:
|
||||
|
||||
```tsx
|
||||
import { useMemo } from "react";
|
||||
import { createTikTokStyleCaptions } from "@remotion/captions";
|
||||
import type { Caption } from "@remotion/captions";
|
||||
|
||||
// How often captions should switch (in milliseconds)
|
||||
// Higher values = more words per page
|
||||
// Lower values = fewer words (more word-by-word)
|
||||
const SWITCH_CAPTIONS_EVERY_MS = 1200;
|
||||
|
||||
const { pages } = useMemo(() => {
|
||||
return createTikTokStyleCaptions({
|
||||
captions,
|
||||
combineTokensWithinMilliseconds: SWITCH_CAPTIONS_EVERY_MS,
|
||||
});
|
||||
}, [captions]);
|
||||
```
|
||||
|
||||
## Rendering with Sequences
|
||||
|
||||
Map over the pages and render each one in a `<Sequence>`. Calculate the start frame and duration from the page timing:
|
||||
|
||||
```tsx
|
||||
import { Sequence, useVideoConfig, AbsoluteFill } from "remotion";
|
||||
import type { TikTokPage } from "@remotion/captions";
|
||||
|
||||
const CaptionedContent: React.FC = () => {
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
return (
|
||||
<AbsoluteFill>
|
||||
{pages.map((page, index) => {
|
||||
const nextPage = pages[index + 1] ?? null;
|
||||
const startFrame = (page.startMs / 1000) * fps;
|
||||
const endFrame = Math.min(
|
||||
nextPage ? (nextPage.startMs / 1000) * fps : Infinity,
|
||||
startFrame + (SWITCH_CAPTIONS_EVERY_MS / 1000) * fps,
|
||||
);
|
||||
const durationInFrames = endFrame - startFrame;
|
||||
|
||||
if (durationInFrames <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Sequence
|
||||
key={index}
|
||||
from={startFrame}
|
||||
durationInFrames={durationInFrames}
|
||||
>
|
||||
<CaptionPage page={page} />
|
||||
</Sequence>
|
||||
);
|
||||
})}
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
## White-space preservation
|
||||
|
||||
The captions are whitespace sensitive. You should include spaces in the `text` field before each word. Use `whiteSpace: "pre"` to preserve the whitespace in the captions.
|
||||
|
||||
## Separate component for captions
|
||||
|
||||
Put captioning logic in a separate component.
|
||||
Make a new file for it.
|
||||
|
||||
## Word highlighting
|
||||
|
||||
A caption page contains `tokens` which you can use to highlight the currently spoken word:
|
||||
|
||||
```tsx
|
||||
import { AbsoluteFill, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import type { TikTokPage } from "@remotion/captions";
|
||||
|
||||
const HIGHLIGHT_COLOR = "#39E508";
|
||||
|
||||
const CaptionPage: React.FC<{ page: TikTokPage }> = ({ page }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
// Current time relative to the start of the sequence
|
||||
const currentTimeMs = (frame / fps) * 1000;
|
||||
// Convert to absolute time by adding the page start
|
||||
const absoluteTimeMs = page.startMs + currentTimeMs;
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ justifyContent: "center", alignItems: "center" }}>
|
||||
<div style={{ fontSize: 80, fontWeight: "bold", whiteSpace: "pre" }}>
|
||||
{page.tokens.map((token) => {
|
||||
const isActive =
|
||||
token.fromMs <= absoluteTimeMs && token.toMs > absoluteTimeMs;
|
||||
|
||||
return (
|
||||
<span
|
||||
key={token.fromMs}
|
||||
style={{ color: isActive ? HIGHLIGHT_COLOR : "white" }}
|
||||
>
|
||||
{token.text}
|
||||
</span>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
## Display captions alongside video content
|
||||
|
||||
By default, put the captions alongside the video content, so the captions are in sync.
|
||||
For each video, make a new captions JSON file.
|
||||
|
||||
```tsx
|
||||
<AbsoluteFill>
|
||||
<Video src={staticFile("video.mp4")} />
|
||||
<CaptionPage page={page} />
|
||||
</AbsoluteFill>
|
||||
```
|
||||
@@ -0,0 +1,69 @@
|
||||
---
|
||||
name: import-srt-captions
|
||||
description: Importing .srt subtitle files into Remotion using @remotion/captions
|
||||
metadata:
|
||||
tags: captions, subtitles, srt, import, parse
|
||||
---
|
||||
|
||||
# Importing .srt subtitles into Remotion
|
||||
|
||||
If you have an existing `.srt` subtitle file, you can import it into Remotion using `parseSrt()` from `@remotion/captions`.
|
||||
|
||||
If you don't have a .srt file, read [Transcribing audio](transcribe-captions.md) for how to generate captions instead.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
First, the @remotion/captions package needs to be installed.
|
||||
If it is not installed, use the following command:
|
||||
|
||||
```bash
|
||||
npx remotion add @remotion/captions # If project uses npm
|
||||
bunx remotion add @remotion/captions # If project uses bun
|
||||
yarn remotion add @remotion/captions # If project uses yarn
|
||||
pnpm exec remotion add @remotion/captions # If project uses pnpm
|
||||
```
|
||||
|
||||
## Reading an .srt file
|
||||
|
||||
Use `staticFile()` to reference an `.srt` file in your `public` folder, then fetch and parse it:
|
||||
|
||||
```tsx
|
||||
import { useState, useEffect, useCallback } from "react";
|
||||
import { AbsoluteFill, staticFile, useDelayRender } from "remotion";
|
||||
import { parseSrt } from "@remotion/captions";
|
||||
import type { Caption } from "@remotion/captions";
|
||||
|
||||
export const MyComponent: React.FC = () => {
|
||||
const [captions, setCaptions] = useState<Caption[] | null>(null);
|
||||
const { delayRender, continueRender, cancelRender } = useDelayRender();
|
||||
const [handle] = useState(() => delayRender());
|
||||
|
||||
const fetchCaptions = useCallback(async () => {
|
||||
try {
|
||||
const response = await fetch(staticFile("subtitles.srt"));
|
||||
const text = await response.text();
|
||||
const { captions: parsed } = parseSrt({ input: text });
|
||||
setCaptions(parsed);
|
||||
continueRender(handle);
|
||||
} catch (e) {
|
||||
cancelRender(e);
|
||||
}
|
||||
}, [continueRender, cancelRender, handle]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchCaptions();
|
||||
}, [fetchCaptions]);
|
||||
|
||||
if (!captions) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return <AbsoluteFill>{/* Use captions here */}</AbsoluteFill>;
|
||||
};
|
||||
```
|
||||
|
||||
Remote URLs are also supported - you can `fetch()` a remote file via URL instead of using `staticFile()`.
|
||||
|
||||
## Using imported captions
|
||||
|
||||
Once parsed, the captions are in the `Caption` format and can be used with all `@remotion/captions` utilities.
|
||||
@@ -0,0 +1,70 @@
|
||||
---
|
||||
name: transcribe-captions
|
||||
description: Transcribing audio to generate captions in Remotion
|
||||
metadata:
|
||||
tags: captions, transcribe, whisper, audio, speech-to-text
|
||||
---
|
||||
|
||||
# Transcribing audio
|
||||
|
||||
To transcribe audio to generate captions in Remotion, you can use the [`transcribe()`](https://www.remotion.dev/docs/install-whisper-cpp/transcribe) function from the [`@remotion/install-whisper-cpp`](https://www.remotion.dev/docs/install-whisper-cpp) package.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
First, the @remotion/install-whisper-cpp package needs to be installed.
|
||||
If it is not installed, use the following command:
|
||||
|
||||
```bash
|
||||
npx remotion add @remotion/install-whisper-cpp
|
||||
```
|
||||
|
||||
## Transcribing
|
||||
|
||||
Make a Node.js script to download Whisper.cpp and a model, and transcribe the audio.
|
||||
|
||||
```ts
|
||||
import path from "path";
|
||||
import {
|
||||
downloadWhisperModel,
|
||||
installWhisperCpp,
|
||||
transcribe,
|
||||
toCaptions,
|
||||
} from "@remotion/install-whisper-cpp";
|
||||
import fs from "fs";
|
||||
|
||||
const to = path.join(process.cwd(), "whisper.cpp");
|
||||
|
||||
await installWhisperCpp({
|
||||
to,
|
||||
version: "1.5.5",
|
||||
});
|
||||
|
||||
await downloadWhisperModel({
|
||||
model: "medium.en",
|
||||
folder: to,
|
||||
});
|
||||
|
||||
// Convert the audio to a 16KHz wav file first if needed:
|
||||
// import {execSync} from 'child_process';
|
||||
// execSync('ffmpeg -i /path/to/audio.mp4 -ar 16000 /path/to/audio.wav -y');
|
||||
|
||||
const whisperCppOutput = await transcribe({
|
||||
model: "medium.en",
|
||||
whisperPath: to,
|
||||
whisperCppVersion: "1.5.5",
|
||||
inputPath: "/path/to/audio123.wav",
|
||||
tokenLevelTimestamps: true,
|
||||
});
|
||||
|
||||
// Optional: Apply our recommended postprocessing
|
||||
const { captions } = toCaptions({
|
||||
whisperCppOutput,
|
||||
});
|
||||
|
||||
// Write it to the public/ folder so it can be fetched from Remotion
|
||||
fs.writeFileSync("captions123.json", JSON.stringify(captions, null, 2));
|
||||
```
|
||||
|
||||
Transcribe each clip individually and create multiple JSON files.
|
||||
|
||||
See [Displaying captions](display-captions.md) for how to display the captions in Remotion.
|
||||
Reference in New Issue
Block a user