PTDT-3807: Add temporal audio annotation support

examples/annotation_import/audio.ipynb

@@ -170,7 +170,7 @@
     },
     {
       "metadata": {},
-      "source": "ontology_builder = lb.OntologyBuilder(classifications=[\n    lb.Classification(class_type=lb.Classification.Type.TEXT,\n                      name=\"text_audio\"),\n    lb.Classification(\n        class_type=lb.Classification.Type.CHECKLIST,\n        name=\"checklist_audio\",\n        options=[\n            lb.Option(value=\"first_checklist_answer\"),\n            lb.Option(value=\"second_checklist_answer\"),\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.RADIO,\n        name=\"radio_audio\",\n        options=[\n            lb.Option(value=\"first_radio_answer\"),\n            lb.Option(value=\"second_radio_answer\"),\n        ],\n    ),\n])\n\nontology = client.create_ontology(\n    \"Ontology Audio Annotations\",\n    ontology_builder.asdict(),\n    media_type=lb.MediaType.Audio,\n)",
+      "source": "ontology_builder = lb.OntologyBuilder(classifications=[\n    # Global (non-temporal) classifications\n    lb.Classification(class_type=lb.Classification.Type.TEXT, name=\"text_audio\"\n                     ),\n    lb.Classification(\n        class_type=lb.Classification.Type.CHECKLIST,\n        name=\"checklist_audio\",\n        options=[\n            lb.Option(value=\"first_checklist_answer\"),\n            lb.Option(value=\"second_checklist_answer\"),\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.RADIO,\n        name=\"radio_audio\",\n        options=[\n            lb.Option(value=\"first_radio_answer\"),\n            lb.Option(value=\"second_radio_answer\"),\n        ],\n    ),\n    # Temporal classifications (scope=INDEX for frame-based annotations)\n    lb.Classification(\n        class_type=lb.Classification.Type.TEXT,\n        name=\"transcription\",\n        scope=lb.Classification.Scope.INDEX,\n        options=[\n            lb.Classification(\n                class_type=lb.Classification.Type.TEXT,\n                name=\"speaker_notes\",\n                options=[\n                    lb.Classification(\n                        class_type=lb.Classification.Type.TEXT,\n                        name=\"context_tags\",\n                    )\n                ],\n            )\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.RADIO,\n        name=\"speaker\",\n        scope=lb.Classification.Scope.INDEX,\n        options=[\n            lb.Option(\"user\"),\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.CHECKLIST,\n        name=\"audio_quality\",\n        scope=lb.Classification.Scope.INDEX,\n        options=[\n            lb.Option(\"background_noise\"),\n            lb.Option(\"echo\"),\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.TEXT,\n        name=\"content_notes\",\n        scope=lb.Classification.Scope.INDEX,\n        options=[\n            lb.Classification(\n                class_type=lb.Classification.Type.RADIO,\n                name=\"clarity_radio\",\n                options=[\n                    lb.Option(\"very_clear\"),\n                    lb.Option(\"slightly_clear\"),\n                ],\n            )\n        ],\n    ),\n    lb.Classification(\n        class_type=lb.Classification.Type.CHECKLIST,\n        name=\"checklist_class\",\n        scope=lb.Classification.Scope.INDEX,\n        options=[\n            lb.Option(\n                \"quality_check\",\n                options=[\n                    lb.Classification(\n                        class_type=lb.Classification.Type.TEXT,\n                        name=\"notes_text\",\n                        options=[\n                            lb.Classification(\n                                class_type=lb.Classification.Type.RADIO,\n                                name=\"severity_radio\",\n                                options=[\n                                    lb.Option(\"minor\"),\n                                ],\n                            )\n                        ],\n                    )\n                ],\n            )\n        ],\n    ),\n])\n\nontology = client.create_ontology(\n    \"Ontology Audio Annotations\",\n    ontology_builder.asdict(),\n    media_type=lb.MediaType.Audio,\n)",
       "cell_type": "code",
       "outputs": [],
       "execution_count": null
@@ -225,7 +225,7 @@
     },
     {
       "metadata": {},
-      "source": "label = []\nlabel.append(\n    lb_types.Label(\n        data={\"global_key\": global_key},\n        annotations=[text_annotation, checklist_annotation, radio_annotation],\n    ))",
+      "source": "label = []\n\n# Regular (global) annotations\nlabel.append(\n    lb_types.Label(\n        data={\"global_key\": global_key},\n        annotations=[text_annotation, checklist_annotation, radio_annotation],\n    ))\n\n# Temporal annotations (using new API)\ntemporal_label = []\ntemporal_label.append(\n    lb_types.Label(\n        data={\"global_key\": global_key},\n        annotations=[\n            temporal_text_annotation,\n            temporal_radio_annotation,\n            temporal_checklist_annotation,\n            nested_text_annotation,\n            inductive_annotation,\n            complex_annotation,\n        ],\n    ))\n\nprint(f\"Created {len(label)} label with regular annotations\")\nprint(\n    f\"Created {len(temporal_label)} label with {len(temporal_label[0].annotations)} temporal annotations\"\n)",
       "cell_type": "code",
       "outputs": [],
       "execution_count": null
@@ -252,6 +252,25 @@
       ],
       "cell_type": "markdown"
     },
+    {
+      "metadata": {},
+      "source": "## Temporal Audio Annotations\n\nLabelbox supports temporal annotations for audio/video with frame-level precision using the new temporal classification API.\n\n### Key Features:\n- **Frame-based timing**: All annotations use millisecond precision\n- **Deep nesting**: Support for multi-level nested classifications (Text > Text > Text, Radio > Radio > Radio, etc.)\n- **Inductive structures**: Multiple parent values can share nested classifications that are automatically split based on frame overlap\n- **Frame validation**: Frames start at 1 (not 0) and must be non-overlapping for Text and Radio siblings\n\n### Important Constraints:\n1. **Frame indexing**: Frames are 1-based (frame 0 is invalid)\n2. **Non-overlapping siblings**: Text and Radio classifications at the same level cannot have overlapping frame ranges\n3. **Overlapping checklists**: Only Checklist answers can have overlapping frame ranges with their siblings",
+      "cell_type": "markdown"
+    },
+    {
+      "metadata": {},
+      "source": "### Example 1: Simple Temporal Text Classification\n\n# Create temporal text annotation with multiple values at different frame ranges\ntemporal_text_annotation = lb_types.TemporalClassificationText(\n    name=\"transcription\",\n    value=[\n        (1000, 1500, \"Hello AI\"),\n        (1501, 2000, \"How are you today?\"),\n    ],\n)\n\nprint(\"Created temporal text annotation with 2 text values\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "metadata": {},
+      "source": "### Example 2: Temporal Radio Question (single answer)\n\n# Create temporal radio annotation with frame range\ntemporal_radio_annotation = lb_types.TemporalClassificationQuestion(\n    name=\"speaker\",\n    value=[\n        lb_types.TemporalClassificationAnswer(\n            name=\"user\",\n            frames=[(1000, 2000)],\n        )\n    ],\n)\n\nprint(\"Created temporal radio annotation\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
     {
       "metadata": {},
       "source": [
@@ -260,6 +279,41 @@
       ],
       "cell_type": "markdown"
     },
+    {
+      "metadata": {},
+      "source": "# Upload temporal annotations via MAL\ntemporal_upload_job = lb.MALPredictionImport.create_from_objects(\n    client=client,\n    project_id=project.uid,\n    name=f\"temporal_mal_job-{str(uuid.uuid4())}\",\n    predictions=temporal_label,  # Use the new temporal_label\n)\n\ntemporal_upload_job.wait_until_done()\nprint(\"Temporal upload completed!\")\nprint(\"Errors:\", temporal_upload_job.errors)\nprint(\"Status:\", temporal_upload_job.statuses)\nprint(\"\\nTemporal annotations uploaded:\")\nprint(\"  - Simple text classification\")\nprint(\"  - Radio classification\")\nprint(\"  - Checklist with overlapping answers\")\nprint(\"  - Nested text (3 levels)\")\nprint(\"  - Inductive structure (shared nested radio)\")\nprint(\"  - Complex nesting (Checklist > Text > Radio)\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "metadata": {},
+      "source": "### Example 4: Nested Temporal Classifications (Text > Text > Text)\n\n# Create deeply nested text classifications\nnested_text_annotation = lb_types.TemporalClassificationText(\n    name=\"transcription\",\n    value=[\n        (1000, 2000, \"Hello, how can I help you?\"),\n    ],\n    classifications=[\n        lb_types.TemporalClassificationText(\n            name=\"speaker_notes\",\n            value=[\n                (1000, 2000, \"Polite greeting\"),\n            ],\n            classifications=[\n                lb_types.TemporalClassificationText(\n                    name=\"context_tags\",\n                    value=[\n                        (1500, 2000, \"customer service tone\"),\n                    ],\n                )\n            ],\n        )\n    ],\n)\n\nprint(\"Created 3-level nested text classification\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "metadata": {},
+      "source": "### Example 5: Inductive Structure (Multiple text values sharing nested classifications)\n\n# This demonstrates an \"inductive structure\" where multiple parent text values\n# share the same nested radio classification. The serializer will automatically\n# split the nested radio so each text value gets only the radio answers that\n# overlap with its frame range.\n\ninductive_annotation = lb_types.TemporalClassificationText(\n    name=\"content_notes\",\n    value=[\n        (1000, 1500, \"Topic is relevant\"),\n        (1501, 2000, \"Good pacing\"),\n    ],\n    classifications=[\n        # This nested radio has answers for BOTH parent text values\n        lb_types.TemporalClassificationQuestion(\n            name=\"clarity_radio\",\n            value=[\n                lb_types.TemporalClassificationAnswer(\n                    name=\"very_clear\",\n                    frames=[(1000, 1500)\n                           ],  # Will be assigned to \"Topic is relevant\"\n                ),\n                lb_types.TemporalClassificationAnswer(\n                    name=\"slightly_clear\",\n                    frames=[(1501, 2000)],  # Will be assigned to \"Good pacing\"\n                ),\n            ],\n        )\n    ],\n)\n\nprint(\"Created inductive structure annotation\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "metadata": {},
+      "source": "### Example 6: Complex Nesting (Checklist > Text > Radio)\n\n# This demonstrates deep nesting with mixed types\ncomplex_annotation = lb_types.TemporalClassificationQuestion(\n    name=\"checklist_class\",\n    value=[\n        lb_types.TemporalClassificationAnswer(\n            name=\"quality_check\",\n            frames=[(1, 1500), (2000, 3000)],\n            classifications=[\n                lb_types.TemporalClassificationText(\n                    name=\"notes_text\",\n                    value=[\n                        (1, 1500, \"Audio quality is excellent\"),\n                        (2000, 2500, \"Some background noise detected\"),\n                    ],\n                    classifications=[\n                        lb_types.TemporalClassificationQuestion(\n                            name=\"severity_radio\",\n                            value=[\n                                lb_types.TemporalClassificationAnswer(\n                                    name=\"minor\",\n                                    frames=[(2000, 2500)],\n                                )\n                            ],\n                        )\n                    ],\n                )\n            ],\n        )\n    ],\n)\n\nprint(\"Created complex nested annotation: Checklist > Text > Radio\")",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "metadata": {},
+      "source": "# Upload temporal annotations via MAL\ntemporal_upload_job = lb.MALPredictionImport.create_from_objects(\n    client=client,\n    project_id=project.uid,\n    name=f\"temporal_mal_job-{str(uuid.uuid4())}\",\n    predictions=label_with_temporal,\n)\n\ntemporal_upload_job.wait_until_done()\nprint(\"Temporal upload completed!\")\nprint(\"Errors:\", temporal_upload_job.errors)\nprint(\"Status:\", temporal_upload_job.statuses)",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
     {
       "metadata": {},
       "source": "# Upload our label using Model-Assisted Labeling\nupload_job = lb.MALPredictionImport.create_from_objects(\n    client=client,\n    project_id=project.uid,\n    name=f\"mal_job-{str(uuid.uuid4())}\",\n    predictions=label,\n)\n\nupload_job.wait_until_done()\nprint(\"Errors:\", upload_job.errors)\nprint(\"Status of uploads: \", upload_job.statuses)",

libs/labelbox/src/labelbox/data/annotation_types/__init__.py

@@ -19,6 +19,10 @@
 from .video import MaskInstance
 from .video import VideoMaskAnnotation
 
+from .temporal import TemporalClassificationText
+from .temporal import TemporalClassificationQuestion
+from .temporal import TemporalClassificationAnswer
+
 from .ner import ConversationEntity
 from .ner import DocumentEntity
 from .ner import DocumentTextSelection
@@ -59,3 +63,70 @@
     MessageRankingTask,
     MessageEvaluationTaskAnnotation,
 )
+
+__all__ = [
+    # Geometry
+    "Line",
+    "Point", 
+    "Mask",
+    "Polygon",
+    "Rectangle",
+    "Geometry",
+    "DocumentRectangle",
+    "RectangleUnit",
+    # Annotation
+    "ClassificationAnnotation",
+    "ObjectAnnotation",
+    # Relationship
+    "RelationshipAnnotation",
+    "Relationship",
+    # Video
+    "VideoClassificationAnnotation",
+    "VideoObjectAnnotation",
+    "MaskFrame",
+    "MaskInstance",
+    "VideoMaskAnnotation",
+    # Temporal
+    "TemporalClassificationText",
+    "TemporalClassificationQuestion",
+    "TemporalClassificationAnswer",
+    # NER
+    "ConversationEntity",
+    "DocumentEntity",
+    "DocumentTextSelection",
+    "TextEntity",
+    # Classification
+    "Checklist",
+    "ClassificationAnswer",
+    "Radio",
+    "Text",
+    # Data
+    "GenericDataRowData",
+    "MaskData",
+    # Label
+    "Label",
+    "LabelGenerator",
+    # Metrics
+    "ScalarMetric",
+    "ScalarMetricAggregation",
+    "ConfusionMatrixMetric",
+    "ConfusionMatrixAggregation",
+    "ScalarMetricValue",
+    "ConfusionMatrixMetricValue",
+    # Tiled Image
+    "EPSG",
+    "EPSGTransformer",
+    "TiledBounds",
+    "TiledImageData",
+    "TileLayer",
+    # LLM Prompt Response
+    "PromptText",
+    "PromptClassificationAnnotation",
+    # MMC
+    "MessageInfo",
+    "OrderedMessageInfo",
+    "MessageSingleSelectionTask",
+    "MessageMultiSelectionTask",
+    "MessageRankingTask",
+    "MessageEvaluationTaskAnnotation",
+]

libs/labelbox/src/labelbox/data/annotation_types/classification/classification.py

@@ -74,3 +74,4 @@ class ClassificationAnnotation(
 
     value: Union[Text, Checklist, Radio]
     message_id: Optional[str] = None
+

libs/labelbox/src/labelbox/data/annotation_types/label.py

@@ -13,6 +13,10 @@
 from .metrics import ScalarMetric, ConfusionMatrixMetric
 from .video import VideoClassificationAnnotation
 from .video import VideoObjectAnnotation, VideoMaskAnnotation
+from .temporal import (
+    TemporalClassificationText,
+    TemporalClassificationQuestion,
+)
 from .mmc import MessageEvaluationTaskAnnotation
 from pydantic import BaseModel, field_validator
 
@@ -44,6 +48,8 @@ class Label(BaseModel):
             ClassificationAnnotation,
             ObjectAnnotation,
             VideoMaskAnnotation,
+            TemporalClassificationText,
+            TemporalClassificationQuestion,
             ScalarMetric,
             ConfusionMatrixMetric,
             RelationshipAnnotation,
@@ -63,8 +69,8 @@ def validate_data(cls, data):
     def object_annotations(self) -> List[ObjectAnnotation]:
         return self._get_annotations_by_type(ObjectAnnotation)
 
-    def classification_annotations(self) -> List[ClassificationAnnotation]:
-        return self._get_annotations_by_type(ClassificationAnnotation)
+    def classification_annotations(self) -> List[Union[ClassificationAnnotation, TemporalClassificationText, TemporalClassificationQuestion]]:
+        return self._get_annotations_by_type((ClassificationAnnotation, TemporalClassificationText, TemporalClassificationQuestion))
 
     def _get_annotations_by_type(self, annotation_type):
         return [
@@ -75,15 +81,43 @@ def _get_annotations_by_type(self, annotation_type):
 
     def frame_annotations(
         self,
-    ) -> Dict[str, Union[VideoObjectAnnotation, VideoClassificationAnnotation]]:
+    ) -> Dict[
+        int,
+        Union[
+            VideoObjectAnnotation,
+            VideoClassificationAnnotation,
+            TemporalClassificationText,
+            TemporalClassificationQuestion,
+        ],
+    ]:
+        """Get temporal annotations organized by frame
+
+        Returns:
+            Dict[int, List]: Dictionary mapping frame (milliseconds) to list of temporal annotations
+
+        Example:
+            >>> label.frame_annotations()
+            {2500: [VideoClassificationAnnotation(...), TemporalClassificationText(...)]}
+
+        Note:
+            For TemporalClassificationText/Question, returns dictionary mapping to start of first frame range.
+            These annotations may have multiple discontinuous frame ranges.
+        """
         frame_dict = defaultdict(list)
         for annotation in self.annotations:
             if isinstance(
                 annotation,
                 (VideoObjectAnnotation, VideoClassificationAnnotation),
             ):
                 frame_dict[annotation.frame].append(annotation)
-        return frame_dict
+            elif isinstance(annotation, (TemporalClassificationText, TemporalClassificationQuestion)):
+                # For temporal annotations with multiple values/answers, use first frame
+                if isinstance(annotation, TemporalClassificationText) and annotation.value:
+                    frame_dict[annotation.value[0][0]].append(annotation)  # value[0][0] is start_frame
+                elif isinstance(annotation, TemporalClassificationQuestion) and annotation.value:
+                    if annotation.value[0].frames:
+                        frame_dict[annotation.value[0].frames[0][0]].append(annotation)  # frames[0][0] is start_frame
+        return dict(frame_dict)
 
     def add_url_to_masks(self, signer) -> "Label":
         """