From 685ba1b4577d94cf6ce0359a8c4d982b465f5507 Mon Sep 17 00:00:00 2001 From: "Michael B. Klein" Date: Thu, 6 Jun 2024 13:35:21 -0500 Subject: [PATCH] Switch from node-webvtt to vtt.js to support styled cues --- package-lock.json | 33 ++++----------- package.json | 6 +-- pnpm-lock.yaml | 22 +++++----- .../Annotation/VTT/Cue.test.tsx | 2 +- .../InformationPanel/Annotation/VTT/Cue.tsx | 12 ++++-- .../Annotation/VTT/VTT.test.tsx | 5 +++ .../InformationPanel/Annotation/VTT/VTT.tsx | 18 ++++----- .../Viewer/InformationPanel/Menu.tsx | 4 +- src/hooks/use-webvtt.ts | 40 +++++++++++++++++-- 9 files changed, 80 insertions(+), 62 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2a6b7f5ef..26e7330c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,14 +24,14 @@ "@stitches/react": "^1.2.8", "flexsearch": "^0.7.43", "hls.js": "^1.5.3", - "node-webvtt": "^1.9.4", "openseadragon": "^4.1.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-error-boundary": "^4.0.12", "sanitize-html": "^2.11.0", - "swiper": "^9.4.1", - "uuid": "^9.0.1" + "swiper": "^9.0.0", + "uuid": "^9.0.1", + "vtt.js": "^0.13.0" }, "devDependencies": { "@iiif/presentation-3": "^1.1.3", @@ -11831,28 +11831,6 @@ "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", "dev": true }, - "node_modules/node-webvtt": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/node-webvtt/-/node-webvtt-1.9.4.tgz", - "integrity": "sha512-EjrJdKdxSyd8j4LMLW6s2Ah4yNoeVXp18Ob04CQl1In18xcUmKzEE8pcsxxnFVqanTyjbGYph2VnvtwIXR4EjA==", - "dependencies": { - "commander": "^7.1.0" - }, - "bin": { - "webvtt-segment": "bin/webvtt-segment.js" - }, - "engines": { - "node": ">= 8.16.0" - } - }, - "node_modules/node-webvtt/node_modules/commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "engines": { - "node": ">= 10" - } - }, "node_modules/non-layered-tidy-tree-layout": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/non-layered-tidy-tree-layout/-/non-layered-tidy-tree-layout-2.0.2.tgz", @@ -15495,6 +15473,11 @@ "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==", "dev": true }, + "node_modules/vtt.js": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/vtt.js/-/vtt.js-0.13.0.tgz", + "integrity": "sha512-t0OJvTmsryCOIHdLWmUPx6V3e0MRGaZMp0Cr5DETWrWNfg9RCHcQlF7X7zNF4mQxhtkyiVKOefdSErSB/KWgOA==" + }, "node_modules/w3c-xmlserializer": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", diff --git a/package.json b/package.json index 75c8451ab..eda4173b2 100644 --- a/package.json +++ b/package.json @@ -87,14 +87,14 @@ "@stitches/react": "^1.2.8", "flexsearch": "^0.7.43", "hls.js": "^1.5.3", - "node-webvtt": "^1.9.4", "openseadragon": "^4.1.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-error-boundary": "^4.0.12", "sanitize-html": "^2.11.0", - "swiper": "^9.4.1", - "uuid": "^9.0.1" + "swiper": "^9.0.0", + "uuid": "^9.0.1", + "vtt.js": "^0.13.0" }, "devDependencies": { "@iiif/presentation-3": "^1.1.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dea02d51f..3ab0518c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,9 +50,6 @@ dependencies: hls.js: specifier: ^1.5.3 version: 1.5.3 - node-webvtt: - specifier: ^1.9.4 - version: 1.9.4 openseadragon: specifier: ^4.1.1 version: 4.1.1 @@ -69,11 +66,14 @@ dependencies: specifier: ^2.11.0 version: 2.11.0 swiper: - specifier: ^9.4.1 + specifier: ^9.0.0 version: 9.4.1 uuid: specifier: ^9.0.1 version: 9.0.1 + vtt.js: + specifier: ^0.13.0 + version: 0.13.0 devDependencies: '@iiif/presentation-3': @@ -2994,6 +2994,7 @@ packages: /commander@7.2.0: resolution: {integrity: sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==} engines: {node: '>= 10'} + dev: true /commander@8.3.0: resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==} @@ -6672,14 +6673,6 @@ packages: resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} dev: true - /node-webvtt@1.9.4: - resolution: {integrity: sha512-EjrJdKdxSyd8j4LMLW6s2Ah4yNoeVXp18Ob04CQl1In18xcUmKzEE8pcsxxnFVqanTyjbGYph2VnvtwIXR4EjA==} - engines: {node: '>= 8.16.0'} - hasBin: true - dependencies: - commander: 7.2.0 - dev: false - /non-layered-tidy-tree-layout@2.0.2: resolution: {integrity: sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw==} dev: true @@ -7416,6 +7409,7 @@ packages: /rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: glob: 7.2.3 @@ -8564,6 +8558,10 @@ packages: resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==} dev: true + /vtt.js@0.13.0: + resolution: {integrity: sha512-t0OJvTmsryCOIHdLWmUPx6V3e0MRGaZMp0Cr5DETWrWNfg9RCHcQlF7X7zNF4mQxhtkyiVKOefdSErSB/KWgOA==} + dev: false + /w3c-xmlserializer@5.0.0: resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} engines: {node: '>=18'} diff --git a/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.test.tsx b/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.test.tsx index c042ac773..e8ea6692b 100644 --- a/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.test.tsx +++ b/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.test.tsx @@ -8,7 +8,7 @@ describe("Information panel cue component", () => { it("renders", () => { render( - + , ); const cue = screen.getByTestId("information-panel-cue"); diff --git a/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.tsx b/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.tsx index a77f4445a..8d00abb22 100644 --- a/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.tsx +++ b/src/components/Viewer/InformationPanel/Annotation/VTT/Cue.tsx @@ -12,7 +12,8 @@ import { const AutoScrollDisableTime = 750; interface Props { - label: string; + html: string; + text: string; start: number; end: number; } @@ -34,7 +35,7 @@ const findScrollableParent = ( return null; }; -const Cue: React.FC = ({ label, start, end }) => { +const Cue: React.FC = ({ html, text, start, end }) => { const dispatch: any = useViewerDispatch(); const { configOptions, @@ -132,9 +133,12 @@ const Cue: React.FC = ({ label, start, end }) => { aria-checked={isActive} data-testid="information-panel-cue" onClick={handleClick} - value={label} + value={text} > - {label} +
{convertTime(start)} ); diff --git a/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.test.tsx b/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.test.tsx index 902f1f0a4..b98abbe74 100644 --- a/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.test.tsx +++ b/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.test.tsx @@ -4,6 +4,10 @@ import AnnotationItemVTT from "./VTT"; import Menu from "src/components/Viewer/InformationPanel/Menu"; import React from "react"; +// Required to prevent an annoying but harmless error while running this test +import { VTTCue } from "vtt.js"; +window.VTTCue = VTTCue; + vi.mock("src/components/Viewer/InformationPanel/Menu"); vi.mocked(Menu).mockReturnValue(
Menu Component
); @@ -49,6 +53,7 @@ describe("AnnotationItemVTT", () => { global.fetch = vitest.fn(() => Promise.reject(new Error("I am the error message")), ); + render(); expect(await screen.findByTestId("error-message")).toHaveTextContent( "Network Error: Error: I am the error message", diff --git a/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.tsx b/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.tsx index d3c9a6981..fde4bdbe3 100644 --- a/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.tsx +++ b/src/components/Viewer/InformationPanel/Annotation/VTT/VTT.tsx @@ -1,14 +1,10 @@ import React, { useEffect } from "react"; -import useWebVtt, { - NodeWebVttCue, - NodeWebVttCueNested, -} from "src/hooks/use-webvtt"; +import useWebVtt, { NodeWebVttCueNested } from "src/hooks/use-webvtt"; import { Group } from "src/components/Viewer/InformationPanel/Annotation/VTT/Cue.styled"; import { InternationalString } from "@iiif/presentation-3"; import Menu from "src/components/Viewer/InformationPanel/Menu"; import { getLabel } from "src/hooks/use-iiif"; -import { parse } from "node-webvtt"; type AnnotationItemVTTProps = { label: InternationalString | undefined; @@ -20,7 +16,7 @@ const AnnotationItemVTT: React.FC = ({ vttUri, }) => { const [cues, setCues] = React.useState>([]); - const { createNestedCues, orderCuesByTime } = useWebVtt(); + const { createNestedCues, orderCuesByTime, parseVttData } = useWebVtt(); const [isNetworkError, setIsNetworkError] = React.useState(); useEffect( @@ -34,11 +30,11 @@ const AnnotationItemVTT: React.FC = ({ }) .then((response) => response.text()) .then((data) => { - const flatCues = parse(data) - .cues as unknown as Array; - const orderedCues = orderCuesByTime(flatCues); - const nestedCues = createNestedCues(orderedCues); - setCues(nestedCues); + parseVttData(data).then((flatCues) => { + const orderedCues = orderCuesByTime(flatCues); + const nestedCues = createNestedCues(orderedCues); + setCues(nestedCues); + }); }) .catch((error) => { console.error(vttUri, error.toString()); diff --git a/src/components/Viewer/InformationPanel/Menu.tsx b/src/components/Viewer/InformationPanel/Menu.tsx index 9f6cb2b6a..9c07f6ef5 100644 --- a/src/components/Viewer/InformationPanel/Menu.tsx +++ b/src/components/Viewer/InformationPanel/Menu.tsx @@ -10,10 +10,10 @@ const Menu: React.FC = ({ items }) => { return ( {items.map((item) => { - const { text, start, end, children, identifier } = item; + const { html, text, start, end, children, identifier } = item; return (
  • - + {children && }
    • ); diff --git a/src/hooks/use-webvtt.ts b/src/hooks/use-webvtt.ts index eca4c2705..b9bd72dee 100644 --- a/src/hooks/use-webvtt.ts +++ b/src/hooks/use-webvtt.ts @@ -1,14 +1,15 @@ // @ts-nocheck import { v4 as uuidv4 } from "uuid"; +import { WebVTT, VTTCue } from "vtt.js"; export interface NodeWebVttCue { identifier?: string; start: number; end: number; + html: string; text: string; - styles?: string; - children?: Array; + align?: "start" | "left" | "center" | "middle" | "end" | "right"; } export interface NodeWebVttCueNested extends NodeWebVttCue { children?: Array; @@ -26,7 +27,7 @@ const useWebVtt = () => { /** * This function takes an array of NodeWebVttCue items as input, where each item - * is an object with properties identifier, start, end, text, and styles. It + * is an object with properties identifier, start, end, html, text, and align. It * iterates through the array of items and uses a stack to keep track of nested * items. It compares the current item's start with the end of the items in the * stack. If the current item's start is smaller than the end of the top item @@ -36,7 +37,9 @@ const useWebVtt = () => { * the nestedItems array. The resulting nestedItems array contains the items * organized into nested structures based on their start and end values. */ - function createNestedCues(flat: Array): Array { + function createNestedCues( + flat: Array, + ): Array { const nestedItems = []; const stack = []; @@ -89,11 +92,40 @@ const useWebVtt = () => { return cues.sort((cue1, cue2) => cue1.start - cue2.start); } + function parseVttData(data: string): Promise> { + return new Promise((resolve, reject) => { + const cues: Array = []; + const parser = new WebVTT.Parser(window, WebVTT.StringDecoder()); + parser.oncue = (cue: VTTCue) => { + const domTree: DocumentFragment = WebVTT.convertCueToDOMTree( + window, + cue.text, + ); + const html = domTree.firstElementChild?.outerHTML || " "; + const text = domTree.firstElementChild?.textContent || ""; + + cues.push({ + identifier: uuidv4(), + start: cue.startTime, + end: cue.endTime, + align: cue.align, + html, + text, + }); + }; + parser.onflush = () => resolve(cues); + parser.onparsingerror = (err) => reject(err); + parser.parse(data); + parser.flush(); + }); + } + return { addIdentifiersToParsedCues, createNestedCues, isChild, orderCuesByTime, + parseVttData, }; };