PTDT-3807: Add temporal audio annotation support

examples/annotation_import/audio.ipynb

@@ -170,7 +170,7 @@
     },
     {
       "metadata": {},
-      "source": "ontology_builder = lb.OntologyBuilder(classifications=[\n    lb.Classification(class_type=lb.Classification.Type.TEXT,\n                      name=\"text_audio\"),\n    lb.Classification(\n        class_type=lb.Classification.Type.CHECKLIST,\n        name=\"checklist_audio\",\n        options=[\n            lb.Option(value=\"first_checklist_answer\"),\n            lb.Option(value=\"second_checklist_answer\"),\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.RADIO,\n        name=\"radio_audio\",\n        options=[\n            lb.Option(value=\"first_radio_answer\"),\n            lb.Option(value=\"second_radio_answer\"),\n        ],\n    ),\n])\n\nontology = client.create_ontology(\n    \"Ontology Audio Annotations\",\n    ontology_builder.asdict(),\n    media_type=lb.MediaType.Audio,\n)",
+      "source": "ontology_builder = lb.OntologyBuilder(classifications=[\n    lb.Classification(class_type=lb.Classification.Type.TEXT,\n                      name=\"text_audio\"),\n    lb.Classification(\n        class_type=lb.Classification.Type.CHECKLIST,\n        name=\"checklist_audio\",\n        options=[\n            lb.Option(value=\"first_checklist_answer\"),\n            lb.Option(value=\"second_checklist_answer\"),\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.RADIO,\n        name=\"radio_audio\",\n        options=[\n            lb.Option(value=\"first_radio_answer\"),\n            lb.Option(value=\"second_radio_answer\"),\n        ],\n    ),\n    # Temporal classification for token-level annotations\n    lb.Classification(\n        class_type=lb.Classification.Type.TEXT,\n        name=\"User Speaker\",\n        scope=lb.Classification.Scope.INDEX,  # INDEX scope for temporal\n    ),\n])\n\nontology = client.create_ontology(\n    \"Ontology Audio Annotations\",\n    ontology_builder.asdict(),\n    media_type=lb.MediaType.Audio,\n)",
       "cell_type": "code",
       "outputs": [],
       "execution_count": null
@@ -252,6 +252,40 @@
       ],
       "cell_type": "markdown"
     },
+    {
+      "metadata": {},
+      "source": [
+        "## Temporal Audio Annotations\n",
+        "\n",
+        "Labelbox supports temporal annotations for audio/video with frame-level precision using the new temporal classification API.\n",
+        "\n",
+        "### Key Features:\n",
+        "- **Frame-based timing**: All annotations use millisecond precision\n",
+        "- **Deep nesting**: Support for multi-level nested classifications (Text > Text > Text, Radio > Radio > Radio, etc.)\n",
+        "- **Inductive structures**: Multiple parent values can share nested classifications that are automatically split based on frame overlap\n",
+        "- **Frame validation**: Frames start at 1 (not 0) and must be non-overlapping for Text and Radio siblings\n",
+        "\n",
+        "### Important Constraints:\n",
+        "1. **Frame indexing**: Frames are 1-based (frame 0 is invalid)\n",
+        "2. **Non-overlapping siblings**: Text and Radio classifications at the same level cannot have overlapping frame ranges\n",
+        "3. **Overlapping checklists**: Only Checklist answers can have overlapping frame ranges with their siblings"
+      ],
+      "cell_type": "markdown"
+    },
+    {
+      "metadata": {},
+      "source": "# Define tokens with precise timing (from demo script)\ntokens_data = [\n    (\"Hello\", 586, 770),  # Hello: frames 586-770\n    (\"AI\", 771, 955),  # AI: frames 771-955\n    (\"how\", 956, 1140),  # how: frames 956-1140\n    (\"are\", 1141, 1325),  # are: frames 1141-1325\n    (\"you\", 1326, 1510),  # you: frames 1326-1510\n    (\"doing\", 1511, 1695),  # doing: frames 1511-1695\n    (\"today\", 1696, 1880),  # today: frames 1696-1880\n]\n\n# Create temporal annotations for each token\ntemporal_annotations = []\nfor token, start_frame, end_frame in tokens_data:\n    token_annotation = lb_types.AudioClassificationAnnotation(\n        frame=start_frame,\n        end_frame=end_frame,\n        name=\"User Speaker\",\n        value=lb_types.Text(answer=token),\n    )\n    temporal_annotations.append(token_annotation)\n\nprint(f\"Created {len(temporal_annotations)} temporal token annotations\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "metadata": {},
+      "source": "# Create label with both regular and temporal annotations\nlabel_with_temporal = []\nlabel_with_temporal.append(\n    lb_types.Label(\n        data={\"global_key\": global_key},\n        annotations=[text_annotation, checklist_annotation, radio_annotation] +\n        temporal_annotations,\n    ))\n\nprint(\n    f\"Created label with {len(label_with_temporal[0].annotations)} total annotations\"\n)\nprint(\"  - Regular annotations: 3\")\nprint(f\"  - Temporal annotations: {len(temporal_annotations)}\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
     {
       "metadata": {},
       "source": [
@@ -260,6 +294,13 @@
       ],
       "cell_type": "markdown"
     },
+    {
+      "metadata": {},
+      "source": "# Upload temporal annotations via MAL\ntemporal_upload_job = lb.MALPredictionImport.create_from_objects(\n    client=client,\n    project_id=project.uid,\n    name=f\"temporal_mal_job-{str(uuid.uuid4())}\",\n    predictions=label_with_temporal,\n)\n\ntemporal_upload_job.wait_until_done()\nprint(\"Temporal upload completed!\")\nprint(\"Errors:\", temporal_upload_job.errors)\nprint(\"Status:\", temporal_upload_job.statuses)",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
     {
       "metadata": {},
       "source": "# Upload our label using Model-Assisted Labeling\nupload_job = lb.MALPredictionImport.create_from_objects(\n    client=client,\n    project_id=project.uid,\n    name=f\"mal_job-{str(uuid.uuid4())}\",\n    predictions=label,\n)\n\nupload_job.wait_until_done()\nprint(\"Errors:\", upload_job.errors)\nprint(\"Status of uploads: \", upload_job.statuses)",

libs/labelbox/src/labelbox/data/annotation_types/__init__.py

@@ -19,6 +19,10 @@
 from .video import MaskInstance
 from .video import VideoMaskAnnotation
 
+from .temporal import TemporalClassificationText
+from .temporal import TemporalClassificationQuestion
+from .temporal import TemporalClassificationAnswer
+
 from .ner import ConversationEntity
 from .ner import DocumentEntity
 from .ner import DocumentTextSelection
@@ -59,3 +63,70 @@
     MessageRankingTask,
     MessageEvaluationTaskAnnotation,
 )
+
+__all__ = [
+    # Geometry
+    "Line",
+    "Point",
+    "Mask",
+    "Polygon",
+    "Rectangle",
+    "Geometry",
+    "DocumentRectangle",
+    "RectangleUnit",
+    # Annotation
+    "ClassificationAnnotation",
+    "ObjectAnnotation",
+    # Relationship
+    "RelationshipAnnotation",
+    "Relationship",
+    # Video
+    "VideoClassificationAnnotation",
+    "VideoObjectAnnotation",
+    "MaskFrame",
+    "MaskInstance",
+    "VideoMaskAnnotation",
+    # Temporal
+    "TemporalClassificationText",
+    "TemporalClassificationQuestion",
+    "TemporalClassificationAnswer",
+    # NER
+    "ConversationEntity",
+    "DocumentEntity",
+    "DocumentTextSelection",
+    "TextEntity",
+    # Classification
+    "Checklist",
+    "ClassificationAnswer",
+    "Radio",
+    "Text",
+    # Data
+    "GenericDataRowData",
+    "MaskData",
+    # Label
+    "Label",
+    "LabelGenerator",
+    # Metrics
+    "ScalarMetric",
+    "ScalarMetricAggregation",
+    "ConfusionMatrixMetric",
+    "ConfusionMatrixAggregation",
+    "ScalarMetricValue",
+    "ConfusionMatrixMetricValue",
+    # Tiled Image
+    "EPSG",
+    "EPSGTransformer",
+    "TiledBounds",
+    "TiledImageData",
+    "TileLayer",
+    # LLM Prompt Response
+    "PromptText",
+    "PromptClassificationAnnotation",
+    # MMC
+    "MessageInfo",
+    "OrderedMessageInfo",
+    "MessageSingleSelectionTask",
+    "MessageMultiSelectionTask",
+    "MessageRankingTask",
+    "MessageEvaluationTaskAnnotation",
+]

libs/labelbox/src/labelbox/data/annotation_types/label.py

@@ -13,6 +13,10 @@
 from .metrics import ScalarMetric, ConfusionMatrixMetric
 from .video import VideoClassificationAnnotation
 from .video import VideoObjectAnnotation, VideoMaskAnnotation
+from .temporal import (
+    TemporalClassificationText,
+    TemporalClassificationQuestion,
+)
 from .mmc import MessageEvaluationTaskAnnotation
 from pydantic import BaseModel, field_validator
 
@@ -44,6 +48,8 @@ class Label(BaseModel):
             ClassificationAnnotation,
             ObjectAnnotation,
             VideoMaskAnnotation,
+            TemporalClassificationText,
+            TemporalClassificationQuestion,
             ScalarMetric,
             ConfusionMatrixMetric,
             RelationshipAnnotation,
@@ -63,8 +69,22 @@ def validate_data(cls, data):
     def object_annotations(self) -> List[ObjectAnnotation]:
         return self._get_annotations_by_type(ObjectAnnotation)
 
-    def classification_annotations(self) -> List[ClassificationAnnotation]:
-        return self._get_annotations_by_type(ClassificationAnnotation)
+    def classification_annotations(
+        self,
+    ) -> List[
+        Union[
+            ClassificationAnnotation,
+            TemporalClassificationText,
+            TemporalClassificationQuestion,
+        ]
+    ]:
+        return self._get_annotations_by_type(
+            (
+                ClassificationAnnotation,
+                TemporalClassificationText,
+                TemporalClassificationQuestion,
+            )
+        )
 
     def _get_annotations_by_type(self, annotation_type):
         return [
@@ -75,15 +95,58 @@ def _get_annotations_by_type(self, annotation_type):
 
     def frame_annotations(
         self,
-    ) -> Dict[str, Union[VideoObjectAnnotation, VideoClassificationAnnotation]]:
+    ) -> Dict[
+        Union[int, None],
+        List[
+            Union[
+                VideoObjectAnnotation,
+                VideoClassificationAnnotation,
+                TemporalClassificationText,
+                TemporalClassificationQuestion,
+            ]
+        ],
+    ]:
+        """Get temporal annotations organized by frame
+
+        Returns:
+            Dict[int, List]: Dictionary mapping frame (milliseconds) to list of temporal annotations
+
+        Example:
+            >>> label.frame_annotations()
+            {2500: [VideoClassificationAnnotation(...), TemporalClassificationText(...)]}
+
+        Note:
+            For TemporalClassificationText/Question, returns dictionary mapping to start of first frame range.
+            These annotations may have multiple discontinuous frame ranges.
+        """
         frame_dict = defaultdict(list)
         for annotation in self.annotations:
             if isinstance(
                 annotation,
                 (VideoObjectAnnotation, VideoClassificationAnnotation),
             ):
                 frame_dict[annotation.frame].append(annotation)
-        return frame_dict
+            elif isinstance(
+                annotation,
+                (TemporalClassificationText, TemporalClassificationQuestion),
+            ):
+                # For temporal annotations with multiple values/answers, use first frame
+                if (
+                    isinstance(annotation, TemporalClassificationText)
+                    and annotation.value
+                ):
+                    frame_dict[annotation.value[0][0]].append(
+                        annotation
+                    )  # value[0][0] is start_frame
+                elif (
+                    isinstance(annotation, TemporalClassificationQuestion)
+                    and annotation.value
+                ):
+                    if annotation.value[0].frames:
+                        frame_dict[annotation.value[0].frames[0][0]].append(
+                            annotation
+                        )  # frames[0][0] is start_frame
+        return dict(frame_dict)
 
     def add_url_to_masks(self, signer) -> "Label":
         """