remotion: New <AnimatedImage> component

packages/core/package.json

@@ -28,6 +28,7 @@
 		"@testing-library/react": "16.1.0",
 		"@testing-library/dom": "10.4.0",
 		"@happy-dom/global-registrator": "14.5.1",
+		"@types/dom-webcodecs": "0.1.11",
 		"happy-dom": "15.10.2",
 		"jsdom": "21.1.0",
 		"react": "19.0.0",

packages/core/src/animated-image/AnimatedImage.tsx

@@ -0,0 +1,142 @@
+import {
+	forwardRef,
+	useEffect,
+	useImperativeHandle,
+	useLayoutEffect,
+	useRef,
+	useState,
+} from 'react';
+import {cancelRender} from '../cancel-render.js';
+import {continueRender, delayRender} from '../delay-render.js';
+import {useCurrentFrame} from '../use-current-frame.js';
+import {useVideoConfig} from '../use-video-config.js';
+import type {AnimatedImageCanvasRef} from './canvas';
+import {Canvas} from './canvas';
+import type {RemotionImageDecoder} from './decode-image.js';
+import {decodeImage} from './decode-image.js';
+import type {RemotionAnimatedImageProps} from './props';
+import {resolveAnimatedImageSource} from './resolve-image-source';
+
+export const AnimatedImage = forwardRef<
+	HTMLCanvasElement,
+	RemotionAnimatedImageProps
+>(
+	(
+		{
+			src,
+			width,
+			height,
+			onError,
+			loopBehavior = 'loop',
+			playbackRate = 1,
+			fit = 'fill',
+			...props
+		},
+		canvasRef,
+	) => {
+		const mountState = useRef({isMounted: true});
+
+		useEffect(() => {
+			const {current} = mountState;
+			current.isMounted = true;
+			return () => {
+				current.isMounted = false;
+			};
+		}, []);
+
+		const resolvedSrc = resolveAnimatedImageSource(src);
+		const [imageDecoder, setImageDecoder] =
+			useState<RemotionImageDecoder | null>(null);
+
+		const [decodeHandle] = useState(() =>
+			delayRender(`Rendering <AnimatedImage/> with src="${resolvedSrc}"`),
+		);
+
+		const frame = useCurrentFrame();
+		const {fps} = useVideoConfig();
+		const currentTime = frame / playbackRate / fps;
+		const currentTimeRef = useRef<number>(currentTime);
+		currentTimeRef.current = currentTime;
+
+		const ref = useRef<AnimatedImageCanvasRef>(null);
+
+		useImperativeHandle(canvasRef, () => {
+			const c = ref.current?.getCanvas();
+			if (!c) {
+				throw new Error('Canvas ref is not set');
+			}
+
+			return c;
+		}, []);
+
+		const [initialLoopBehavior] = useState(() => loopBehavior);
+
+		useEffect(() => {
+			const controller = new AbortController();
+			decodeImage({
+				resolvedSrc,
+				signal: controller.signal,
+				currentTime: currentTimeRef.current,
+				initialLoopBehavior,
+			})
+				.then((d) => {
+					setImageDecoder(d);
+					continueRender(decodeHandle);
+				})
+				.catch((err) => {
+					if ((err as Error).name === 'AbortError') {
+						continueRender(decodeHandle);
+
+						return;
+					}
+
+					if (onError) {
+						onError?.(err as Error);
+						continueRender(decodeHandle);
+					} else {
+						cancelRender(err);
+					}
+				});
+
+			return () => {
+				controller.abort();
+			};
+		}, [resolvedSrc, decodeHandle, onError, initialLoopBehavior]);
+
+		useLayoutEffect(() => {
+			if (!imageDecoder) {
+				return;
+			}
+
+			const delay = delayRender(
+				`Rendering frame at ${currentTime} of <AnimatedImage src="${src}"/>`,
+			);
+
+			imageDecoder
+				.getFrame(currentTime, loopBehavior)
+				.then((videoFrame) => {
+					if (mountState.current.isMounted) {
+						if (videoFrame === null) {
+							ref.current?.clear();
+						} else {
+							ref.current?.draw(videoFrame.frame!);
+						}
+					}
+
+					continueRender(delay);
+				})
+				.catch((err) => {
+					if (onError) {
+						onError(err as Error);
+						continueRender(delay);
+					} else {
+						cancelRender(err);
+					}
+				});
+		}, [currentTime, imageDecoder, loopBehavior, onError, src]);
+
+		return (
+			<Canvas ref={ref} width={width} height={height} fit={fit} {...props} />
+		);
+	},
+);

packages/core/src/animated-image/canvas.tsx

@@ -0,0 +1,163 @@
+import React, {useCallback, useImperativeHandle, useRef} from 'react';
+import type {AnimatedImageFillMode} from './props';
+
+const calcArgs = (
+	fit: AnimatedImageFillMode,
+	frameSize: {
+		width: number;
+		height: number;
+	},
+	canvasSize: {
+		width: number;
+		height: number;
+	},
+): [number, number, number, number, number, number, number, number] => {
+	switch (fit) {
+		case 'fill': {
+			return [
+				0,
+				0,
+				frameSize.width,
+				frameSize.height,
+				0,
+				0,
+				canvasSize.width,
+				canvasSize.height,
+			];
+		}
+
+		case 'contain': {
+			const ratio = Math.min(
+				canvasSize.width / frameSize.width,
+				canvasSize.height / frameSize.height,
+			);
+
+			const centerX = (canvasSize.width - frameSize.width * ratio) / 2;
+			const centerY = (canvasSize.height - frameSize.height * ratio) / 2;
+
+			return [
+				0,
+				0,
+				frameSize.width,
+				frameSize.height,
+				centerX,
+				centerY,
+				frameSize.width * ratio,
+				frameSize.height * ratio,
+			];
+		}
+
+		case 'cover': {
+			const ratio = Math.max(
+				canvasSize.width / frameSize.width,
+				canvasSize.height / frameSize.height,
+			);
+			const centerX = (canvasSize.width - frameSize.width * ratio) / 2;
+			const centerY = (canvasSize.height - frameSize.height * ratio) / 2;
+
+			return [
+				0,
+				0,
+				frameSize.width,
+				frameSize.height,
+				centerX,
+				centerY,
+				frameSize.width * ratio,
+				frameSize.height * ratio,
+			];
+		}
+
+		default:
+			throw new Error('Unknown fit: ' + fit);
+	}
+};
+
+type Props = {
+	readonly width?: number;
+
+	readonly height?: number;
+	readonly fit: AnimatedImageFillMode;
+
+	readonly className?: string;
+
+	readonly style?: React.CSSProperties;
+};
+
+export type AnimatedImageCanvasRef = {
+	readonly draw: (imageData: VideoFrame) => void;
+	readonly getCanvas: () => HTMLCanvasElement | null;
+	clear: () => void;
+};
+
+const CanvasRefForwardingFunction: React.ForwardRefRenderFunction<
+	AnimatedImageCanvasRef,
+	Props
+> = ({width, height, fit, className, style}, ref) => {
+	const canvasRef = useRef<HTMLCanvasElement>(null);
+
+	const draw = useCallback(
+		(imageData: VideoFrame) => {
+			const canvas = canvasRef.current;
+			const canvasWidth = width ?? imageData.displayWidth;
+			const canvasHeight = height ?? imageData.displayHeight;
+
+			if (!canvas) {
+				throw new Error('Canvas ref is not set');
+			}
+
+			const ctx = canvasRef.current?.getContext('2d');
+			if (!ctx) {
+				throw new Error('Could not get 2d context');
+			}
+
+			canvas.width = canvasWidth;
+			canvas.height = canvasHeight;
+
+			ctx.drawImage(
+				imageData,
+				...calcArgs(
+					fit,
+					{
+						height: imageData.displayHeight,
+						width: imageData.displayWidth,
+					},
+					{
+						width: canvasWidth,
+						height: canvasHeight,
+					},
+				),
+			);
+		},
+		[fit, height, width],
+	);
+
+	useImperativeHandle(ref, () => {
+		return {
+			draw,
+			getCanvas: () => {
+				if (!canvasRef.current) {
+					throw new Error('Canvas ref is not set');
+				}
+
+				return canvasRef.current;
+			},
+			clear: () => {
+				const ctx = canvasRef.current?.getContext('2d');
+				if (!ctx) {
+					throw new Error('Could not get 2d context');
+				}
+
+				ctx.clearRect(
+					0,
+					0,
+					canvasRef.current!.width,
+					canvasRef.current!.height,
+				);
+			},
+		};
+	}, [draw]);
+
+	return <canvas ref={canvasRef} className={className} style={style} />;
+};
+
+export const Canvas = React.forwardRef(CanvasRefForwardingFunction);