From 2b4a3a97a812b6e7e607532e89037ff7c792c301 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 11 Dec 2024 11:18:45 -0800 Subject: [PATCH 01/59] draft --- docs/contributing/migration.md | 46 +++--- examples/experimental/otel_exporter.ipynb | 156 ++++++++++++++++++ .../trulens/core/database/migrations/data.py | 2 +- .../versions/10_create_event_table.py | 60 +++++++ src/core/trulens/core/database/orm.py | 42 +++++ src/core/trulens/core/schema/event.py | 52 ++++++ src/core/trulens/core/schema/types.py | 4 + 7 files changed, 338 insertions(+), 24 deletions(-) create mode 100644 examples/experimental/otel_exporter.ipynb create mode 100644 src/core/trulens/core/database/migrations/versions/10_create_event_table.py create mode 100644 src/core/trulens/core/schema/event.py diff --git a/docs/contributing/migration.md b/docs/contributing/migration.md index a2f3badbe..a492aef6f 100644 --- a/docs/contributing/migration.md +++ b/docs/contributing/migration.md @@ -8,13 +8,13 @@ schema. If upgrading DB, You must do this step!! 1. Make desired changes to SQLAlchemy orm models in `src/core/trulens/core/database/orm.py`. -1. Get a database with the new changes: - 1. `rm default.sqlite` - 1. Run `TruSession()` to create a fresh database that uses the new ORM. 1. Run automatic alembic revision script generator. This will generate a new python script in `src/core/trulens/core/database/migrations`. 1. `cd src/core/trulens/core/database/migrations` 1. `SQLALCHEMY_URL="sqlite:///../../../../../../default.sqlite" alembic revision --autogenerate -m "" --rev-id ""` 1. Check over the automatically generated script in `src/core/trulens/core/database/migration/versions` to make sure it looks correct. +1. Get a database with the new changes: + 1. `rm default.sqlite` + 1. Run `TruSession()` to create a fresh database that uses the new ORM. 1. Add the version to `src/core/trulens/core/database/migrations/data.py` in the variable `sql_alchemy_migration_versions` 1. Make any `sqlalchemy_upgrade_paths` updates in `src/core/trulens/core/database/migrations/data.py` if a backfill is necessary. @@ -30,32 +30,32 @@ Note: Some of these instructions may be outdated and are in progress if being up github; which will invalidate it upon commit) 1. cd `tests/docs_notebooks/notebooks_to_test` 1. remove any local dbs - * `rm -rf default.sqlite` + - `rm -rf default.sqlite` 1. run below notebooks (Making sure you also run with the most recent code in trulens) TODO: Move these to a script - * all_tools.ipynb # `cp ../../../generated_files/all_tools.ipynb ./` - * llama_index_quickstart.ipynb # `cp - ../../../examples/quickstart/llama_index_quickstart.ipynb ./` - * langchain-retrieval-augmentation-with-trulens.ipynb # `cp - ../../../examples/vector-dbs/pinecone/langchain-retrieval-augmentation-with-trulens.ipynb - ./` - * Add any other notebooks you think may have possible breaking changes + - all_tools.ipynb # `cp ../../../generated_files/all_tools.ipynb ./` + - llama_index_quickstart.ipynb # `cp +../../../examples/quickstart/llama_index_quickstart.ipynb ./` + - langchain-retrieval-augmentation-with-trulens.ipynb # `cp +../../../examples/vector-dbs/pinecone/langchain-retrieval-augmentation-with-trulens.ipynb +./` + - Add any other notebooks you think may have possible breaking changes 1. replace the last compatible db with this new db file - * Use the version you chose for --rev-id - * `mkdir release_dbs/sql_alchemy_/` - * `cp default.sqlite - release_dbs/sql_alchemy_/` + - Use the version you chose for --rev-id + - `mkdir release_dbs/sql_alchemy_/` + - `cp default.sqlite +release_dbs/sql_alchemy_/` 1. `git add release_dbs` ## Testing the DB Run the tests with the requisite env vars. - ```bash - HUGGINGFACE_API_KEY="" \ - OPENAI_API_KEY="" \ - PINECONE_API_KEY="" \ - PINECONE_ENV="" \ - HUGGINGFACEHUB_API_TOKEN="" \ - python -m pytest tests/docs_notebooks -k backwards_compat - ``` +```bash +HUGGINGFACE_API_KEY="" \ +OPENAI_API_KEY="" \ +PINECONE_API_KEY="" \ +PINECONE_ENV="" \ +HUGGINGFACEHUB_API_TOKEN="" \ +python -m pytest tests/docs_notebooks -k backwards_compat +``` diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb new file mode 100644 index 000000000..4b6c16a96 --- /dev/null +++ b/examples/experimental/otel_exporter.ipynb @@ -0,0 +1,156 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install opentelemetry-api\n", + "# !pip install opentelemetry-sdk" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Add base dir to path to be able to access test folder.\n", + "base_dir = Path().cwd().parent.parent.resolve()\n", + "if str(base_dir) not in sys.path:\n", + " print(f\"Adding {base_dir} to sys.path\")\n", + " sys.path.append(str(base_dir))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from opentelemetry import trace\n", + "from opentelemetry.sdk.resources import Resource\n", + "from opentelemetry.sdk.trace import TracerProvider\n", + "from opentelemetry.sdk.trace.export import BatchSpanProcessor\n", + "from opentelemetry.sdk.trace.export import ConsoleSpanExporter\n", + "\n", + "SERVICE_NAME = \"trulens\"\n", + "\n", + "\n", + "def init():\n", + " # Use Resource.create() instead of constructor directly\n", + " resource = Resource.create({\"service.name\": SERVICE_NAME})\n", + "\n", + " trace.set_tracer_provider(TracerProvider(resource=resource))\n", + " trace.get_tracer_provider().add_span_processor(\n", + " BatchSpanProcessor(ConsoleSpanExporter())\n", + " )\n", + "\n", + "\n", + "init()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Callable\n", + "\n", + "from trulens.apps.custom import instrument\n", + "\n", + "SERVICE_NAME = \"trulens\"\n", + "\n", + "\n", + "def decorator(func: Callable):\n", + " tracer = trace.get_tracer(SERVICE_NAME)\n", + "\n", + " def wrapper(*args, **kwargs):\n", + " print(\"start wrap\")\n", + "\n", + " with tracer.start_as_current_span(\"custom\"):\n", + " result = func(*args, **kwargs)\n", + " span = trace.get_current_span()\n", + " print(\"---span---\")\n", + " print(span.get_span_context())\n", + " span.set_attribute(\"result\", result)\n", + " span.set_status(trace.Status(trace.StatusCode.OK))\n", + " return result\n", + "\n", + " return wrapper" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from examples.dev.dummy_app.dummy import Dummy\n", + "\n", + "\n", + "class TestApp(Dummy):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " @decorator\n", + " @instrument\n", + " def respond_to_query(self, query: str) -> str:\n", + " return f\"answer: {self.nested(query)}\"\n", + "\n", + " @decorator\n", + " @instrument\n", + " def nested(self, query: str) -> str:\n", + " return f\"nested: {query}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.core.session import TruSession\n", + "\n", + "SERVICE_NAME = \"trulens\"\n", + "\n", + "\n", + "# session = TruSession()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session = TruSession()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "trulens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/core/trulens/core/database/migrations/data.py b/src/core/trulens/core/database/migrations/data.py index 2c0dd3274..f5a6ea1dd 100644 --- a/src/core/trulens/core/database/migrations/data.py +++ b/src/core/trulens/core/database/migrations/data.py @@ -15,7 +15,7 @@ from trulens.core.schema import record as record_schema from trulens.core.utils import pyschema as pyschema_utils -sql_alchemy_migration_versions: List[int] = [1, 2, 3] +sql_alchemy_migration_versions: List[int] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] """DB versions.""" sqlalchemy_upgrade_paths: Dict[int, Tuple[int, Callable[[DB]]]] = { diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py new file mode 100644 index 000000000..fe7af0340 --- /dev/null +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -0,0 +1,60 @@ +"""create event table + +Revision ID: 10 +Revises: 9 +Create Date: 2024-12-11 09:32:48.976169 +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "10" +down_revision = "9" +branch_labels = None +depends_on = None + + +def upgrade(config) -> None: + prefix = config.get_main_option("trulens.table_prefix") + + if prefix is None: + raise RuntimeError("trulens.table_prefix is not set") + + # TODO: The automatically generated code below likely references + # tables such as "trulens_feedback_defs" or "trulens_records". + # However, the common prefix for these tables "trulens_" is + # actually configurable and so replace it with the variable + # prefix. + # e.g. replace "trulens_records" with prefix + "records". + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "trulens_events", + sa.Column("event_id", sa.VARCHAR(length=256), nullable=False), + sa.Column("record", sa.Text(), nullable=False), + sa.Column("record_attributes", sa.Text(), nullable=False), + sa.Column("record_type", sa.VARCHAR(length=256), nullable=False), + sa.Column("resource_attributes", sa.Text(), nullable=False), + sa.Column("start_timestamp", sa.Float(), nullable=False), + sa.Column("timestamp", sa.Float(), nullable=False), + sa.Column("trace", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("event_id"), + ) + # ### end Alembic commands ### + + +def downgrade(config) -> None: + prefix = config.get_main_option("trulens.table_prefix") + + if prefix is None: + raise RuntimeError("trulens.table_prefix is not set") + + # TODO: The automatically generated code below likely references + # tables such as "trulens_feedback_defs" or "trulens_records". + # However, the common prefix for these tables "trulens_" is + # actually configurable and so replace it with the variable + # prefix. + # e.g. replace "trulens_records" with prefix + "records". + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("trulens_events") + # ### end Alembic commands ### diff --git a/src/core/trulens/core/database/orm.py b/src/core/trulens/core/database/orm.py index 67a2dec75..0740072cb 100644 --- a/src/core/trulens/core/database/orm.py +++ b/src/core/trulens/core/database/orm.py @@ -22,6 +22,7 @@ from trulens.core.database import base as core_db from trulens.core.schema import app as app_schema from trulens.core.schema import dataset as dataset_schema +from trulens.core.schema import event as event_schema from trulens.core.schema import feedback as feedback_schema from trulens.core.schema import groundtruth as groundtruth_schema from trulens.core.schema import record as record_schema @@ -116,6 +117,7 @@ class ORM(abc.ABC, Generic[T]): FeedbackResult: Type[T] GroundTruth: Type[T] Dataset: Type[T] + EventTable: Type[T] def new_orm(base: Type[T], prefix: str = "trulens_") -> Type[ORM[T]]: @@ -401,6 +403,46 @@ def parse( dataset_json=obj.model_dump_json(redact_keys=redact_keys), ) + class EventTable(base): + """ + ORM class for OTEL traces/spans. + """ + + _table_base_name = "events" + + event_id = Column(TYPE_ID, nullable=False, primary_key=True) + record = Column(TYPE_JSON, nullable=False) + record_attributes = Column(TYPE_JSON, nullable=False) + record_type = Column(VARCHAR(256), nullable=False) + resource_attributes = Column(TYPE_JSON, nullable=False) + start_timestamp = Column(TYPE_TIMESTAMP, nullable=False) + timestamp = Column(TYPE_TIMESTAMP, nullable=False) + trace = Column(TYPE_JSON, nullable=False) + + @classmethod + def parse( + cls, + obj: event_schema.Event, + ) -> ORM.EventTable: + return cls( + event_id=obj.event_id, + record=json_utils.json_str_of_obj( + obj.record, redact_keys=True + ), + record_attributes=json_utils.json_str_of_obj( + obj.record_attributes, redact_keys=True + ), + record_type=obj.record_type, + resource_attributes=json_utils.json_str_of_obj( + obj.resource_attributes, redact_keys=True + ), + start_timestamp=obj.start_timestamp, + timestamp=obj.timestamp, + trace=json_utils.json_str_of_obj( + obj.trace, redact_keys=True + ), + ) + configure_mappers() # IMPORTANT # Without the above, orm class attributes which are defined using backref # will not be visible, i.e. orm.AppDefinition.records. diff --git a/src/core/trulens/core/schema/event.py b/src/core/trulens/core/schema/event.py new file mode 100644 index 000000000..3076deb3a --- /dev/null +++ b/src/core/trulens/core/schema/event.py @@ -0,0 +1,52 @@ +"""Serializable event-related classes.""" + +from __future__ import annotations + +import datetime +import logging +from typing import Any, Dict, Hashable, Optional + +import pydantic +from trulens.core.schema import types as types_schema +from trulens.core.utils import serial as serial_utils + +logger = logging.getLogger(__name__) + + +class Event(serial_utils.SerialModel, Hashable): + """The class that represents a single event data entry.""" + + event_id: types_schema.EventID # str + """The unique identifier for the event.""" + + record: Dict[str, Any] + record_attributes: Dict[str, Any] + record_type: str + resource_attributes: Dict[str, Any] + start_timestamp: datetime.datetime = pydantic.Field( + default_factory=datetime.datetime.now + ) + + timestamp: datetime.datetime = pydantic.Field( + default_factory=datetime.datetime.now + ) + + trace: Dict[str, Any] + + def __init__( + self, + event_id: types_schema.EventID, + event_type: str, + event_time: datetime.datetime, + event_data: Optional[Dict] = None, + meta: Optional[types_schema.Metadata] = None, + **kwargs, + ): + kwargs["event_id"] = event_id + kwargs["event_type"] = event_type + kwargs["event_time"] = event_time + kwargs["event_data"] = event_data + kwargs["meta"] = meta if meta is not None else {} + + def __hash__(self): + return hash(self.event_id) diff --git a/src/core/trulens/core/schema/types.py b/src/core/trulens/core/schema/types.py index 18428c715..5384bb34f 100644 --- a/src/core/trulens/core/schema/types.py +++ b/src/core/trulens/core/schema/types.py @@ -83,3 +83,7 @@ def new_call_id() -> CallID: By default these are hashes of dataset content as json. """ + +EventID: TypeAlias = str +"""Unique identifier for a event. +""" From 49d79dc313f3060e3cf4fec9ce28d651581d72c1 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 12 Dec 2024 13:54:55 -0800 Subject: [PATCH 02/59] update --- .../versions/10_create_event_table.py | 4 +- src/core/trulens/core/database/orm.py | 59 ++++++++++++++++--- src/core/trulens/core/schema/event.py | 56 ++++++++++++------ 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index fe7af0340..57ea4cc23 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -35,8 +35,8 @@ def upgrade(config) -> None: sa.Column("record_attributes", sa.Text(), nullable=False), sa.Column("record_type", sa.VARCHAR(length=256), nullable=False), sa.Column("resource_attributes", sa.Text(), nullable=False), - sa.Column("start_timestamp", sa.Float(), nullable=False), - sa.Column("timestamp", sa.Float(), nullable=False), + sa.Column("start_timestamp", sa.TIMESTAMP(), nullable=False), + sa.Column("timestamp", sa.TIMESTAMP(), nullable=False), sa.Column("trace", sa.Text(), nullable=False), sa.PrimaryKeyConstraint("event_id"), ) diff --git a/src/core/trulens/core/database/orm.py b/src/core/trulens/core/database/orm.py index 0740072cb..ee50b172b 100644 --- a/src/core/trulens/core/database/orm.py +++ b/src/core/trulens/core/database/orm.py @@ -5,6 +5,7 @@ from sqlite3 import Connection as SQLite3Connection from typing import ClassVar, Dict, Generic, Type, TypeVar +from sqlalchemy import DATETIME from sqlalchemy import VARCHAR from sqlalchemy import Column from sqlalchemy import Engine @@ -117,7 +118,7 @@ class ORM(abc.ABC, Generic[T]): FeedbackResult: Type[T] GroundTruth: Type[T] Dataset: Type[T] - EventTable: Type[T] + Event: Type[T] def new_orm(base: Type[T], prefix: str = "trulens_") -> Type[ORM[T]]: @@ -403,43 +404,83 @@ def parse( dataset_json=obj.model_dump_json(redact_keys=redact_keys), ) - class EventTable(base): + class Event(base): """ - ORM class for OTEL traces/spans. + ORM class for OTEL traces/spans. This should be kept in line with the event table + https://docs.snowflake.com/en/developer-guide/logging-tracing/event-table-columns#data-for-trace-events + https://docs.snowflake.com/en/developer-guide/logging-tracing/event-table-columns#label-event-table-record-column-span """ _table_base_name = "events" event_id = Column(TYPE_ID, nullable=False, primary_key=True) + """ + The unique identifier for the event. This is just the span_id. + """ + record = Column(TYPE_JSON, nullable=False) + """ + For a span, this is an object that includes: + - name: the function/procedure that emitted the data + - kind: SPAN_KIND_TRULENS + - parent_span_id: the unique identifier for the parent span + - status: STATUS_CODE_ERROR when the span corresponds to an unhandled exception. Otherwise, STATUS_CODE_UNSET. + """ + record_attributes = Column(TYPE_JSON, nullable=False) + """ + Attributes of the record that can either come from the user, or based on the TruLens semantic conventions. + """ + record_type = Column(VARCHAR(256), nullable=False) + """ + Specifies the kind of record specified by this row. This will always be "SPAN" for TruLens. + """ + resource_attributes = Column(TYPE_JSON, nullable=False) - start_timestamp = Column(TYPE_TIMESTAMP, nullable=False) - timestamp = Column(TYPE_TIMESTAMP, nullable=False) + """ + Reserved. + """ + + start_timestamp = Column(DATETIME, nullable=False) + """ + The timestamp when the span started. This is a UNIX timestamp in milliseconds. + Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. + """ + + timestamp = Column(DATETIME, nullable=False) + """ + The timestamp when the span concluded. This is a UNIX timestamp in milliseconds. + Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. + """ + trace = Column(TYPE_JSON, nullable=False) + """ + Contains the span context, including the trace_id and span_id for the span. + """ @classmethod def parse( cls, obj: event_schema.Event, + redact_keys: bool = False, ) -> ORM.EventTable: return cls( event_id=obj.event_id, record=json_utils.json_str_of_obj( - obj.record, redact_keys=True + obj.record, redact_keys=redact_keys ), record_attributes=json_utils.json_str_of_obj( - obj.record_attributes, redact_keys=True + obj.record_attributes, redact_keys=redact_keys ), record_type=obj.record_type, resource_attributes=json_utils.json_str_of_obj( - obj.resource_attributes, redact_keys=True + obj.resource_attributes, redact_keys=redact_keys ), start_timestamp=obj.start_timestamp, timestamp=obj.timestamp, trace=json_utils.json_str_of_obj( - obj.trace, redact_keys=True + obj.trace, redact_keys=redact_keys ), ) diff --git a/src/core/trulens/core/schema/event.py b/src/core/trulens/core/schema/event.py index 3076deb3a..5a21f326f 100644 --- a/src/core/trulens/core/schema/event.py +++ b/src/core/trulens/core/schema/event.py @@ -2,11 +2,10 @@ from __future__ import annotations -import datetime +from datetime import datetime import logging -from typing import Any, Dict, Hashable, Optional +from typing import Any, Dict, Hashable -import pydantic from trulens.core.schema import types as types_schema from trulens.core.utils import serial as serial_utils @@ -17,36 +16,55 @@ class Event(serial_utils.SerialModel, Hashable): """The class that represents a single event data entry.""" event_id: types_schema.EventID # str - """The unique identifier for the event.""" + """ + The unique identifier for the event. This is just the span_id. + """ record: Dict[str, Any] + """ + For a span, this is an object that includes: + - name: the function/procedure that emitted the data + - kind: SPAN_KIND_TRULENS + - parent_span_id: the unique identifier for the parent span + - status: STATUS_CODE_ERROR when the span corresponds to an unhandled exception. Otherwise, STATUS_CODE_UNSET. + """ + record_attributes: Dict[str, Any] + """ + Attributes of the record that can either come from the user, or based on the TruLens semantic conventions. + """ + record_type: str + """ + Specifies the kind of record specified by this row. This will always be "SPAN" for TruLens. + """ + resource_attributes: Dict[str, Any] - start_timestamp: datetime.datetime = pydantic.Field( - default_factory=datetime.datetime.now - ) + """ + Reserved. + """ + + start_timestamp: datetime + """ + The timestamp when the span started. This is a UNIX timestamp in milliseconds. + Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. + """ - timestamp: datetime.datetime = pydantic.Field( - default_factory=datetime.datetime.now - ) + timestamp: datetime + """ + The timestamp when the span concluded. This is a UNIX timestamp in milliseconds. + Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. + """ trace: Dict[str, Any] def __init__( self, event_id: types_schema.EventID, - event_type: str, - event_time: datetime.datetime, - event_data: Optional[Dict] = None, - meta: Optional[types_schema.Metadata] = None, **kwargs, ): - kwargs["event_id"] = event_id - kwargs["event_type"] = event_type - kwargs["event_time"] = event_time - kwargs["event_data"] = event_data - kwargs["meta"] = meta if meta is not None else {} + super().__init__(event_id=event_id, **kwargs) + self.event_id = event_id def __hash__(self): return hash(self.event_id) From 7207252be2cf600c6ad790576daf62a63d165359 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Tue, 17 Dec 2024 10:15:35 -0500 Subject: [PATCH 03/59] prefix --- .../versions/10_create_event_table.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index 57ea4cc23..4fa7ce498 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -21,15 +21,9 @@ def upgrade(config) -> None: if prefix is None: raise RuntimeError("trulens.table_prefix is not set") - # TODO: The automatically generated code below likely references - # tables such as "trulens_feedback_defs" or "trulens_records". - # However, the common prefix for these tables "trulens_" is - # actually configurable and so replace it with the variable - # prefix. - # e.g. replace "trulens_records" with prefix + "records". # ### commands auto generated by Alembic - please adjust! ### op.create_table( - "trulens_events", + prefix + "events", sa.Column("event_id", sa.VARCHAR(length=256), nullable=False), sa.Column("record", sa.Text(), nullable=False), sa.Column("record_attributes", sa.Text(), nullable=False), @@ -49,12 +43,8 @@ def downgrade(config) -> None: if prefix is None: raise RuntimeError("trulens.table_prefix is not set") - # TODO: The automatically generated code below likely references - # tables such as "trulens_feedback_defs" or "trulens_records". - # However, the common prefix for these tables "trulens_" is - # actually configurable and so replace it with the variable - # prefix. - # e.g. replace "trulens_records" with prefix + "records". # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("trulens_events") + op.drop_table( + prefix + "events", + ) # ### end Alembic commands ### From f4330191320b93114b4782222914153f7e9a43a7 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 16:36:05 -0500 Subject: [PATCH 04/59] updates --- .../versions/10_create_event_table.py | 31 +++++++-------- src/core/trulens/core/database/orm.py | 38 +++++++++++-------- src/core/trulens/core/schema/event.py | 35 +++++++++++------ 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index 4fa7ce498..3fea460ad 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -6,6 +6,9 @@ """ from alembic import op +from snowflake.sqlalchemy import OBJECT +from snowflake.sqlalchemy import TIMESTAMP_NTZ +from snowflake.sqlalchemy import dialect as SnowflakeDialect import sqlalchemy as sa # revision identifiers, used by Alembic. @@ -22,18 +25,17 @@ def upgrade(config) -> None: raise RuntimeError("trulens.table_prefix is not set") # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - prefix + "events", - sa.Column("event_id", sa.VARCHAR(length=256), nullable=False), - sa.Column("record", sa.Text(), nullable=False), - sa.Column("record_attributes", sa.Text(), nullable=False), - sa.Column("record_type", sa.VARCHAR(length=256), nullable=False), - sa.Column("resource_attributes", sa.Text(), nullable=False), - sa.Column("start_timestamp", sa.TIMESTAMP(), nullable=False), - sa.Column("timestamp", sa.TIMESTAMP(), nullable=False), - sa.Column("trace", sa.Text(), nullable=False), - sa.PrimaryKeyConstraint("event_id"), - ) + if op.get_context().dialect.name == SnowflakeDialect.name: + op.create_table( + prefix + "events", + sa.Column("record", OBJECT(), nullable=False), + sa.Column("record_attributes", OBJECT(), nullable=False), + sa.Column("record_type", sa.VARCHAR(length=256), nullable=False), + sa.Column("resource_attributes", OBJECT(), nullable=False), + sa.Column("start_timestamp", TIMESTAMP_NTZ(), nullable=False), + sa.Column("timestamp", TIMESTAMP_NTZ(), nullable=False), + sa.Column("trace", OBJECT(), nullable=False), + ) # ### end Alembic commands ### @@ -44,7 +46,6 @@ def downgrade(config) -> None: raise RuntimeError("trulens.table_prefix is not set") # ### commands auto generated by Alembic - please adjust! ### - op.drop_table( - prefix + "events", - ) + if op.get_context().dialect.name == SnowflakeDialect.name: + op.drop_table(prefix + "events") # ### end Alembic commands ### diff --git a/src/core/trulens/core/database/orm.py b/src/core/trulens/core/database/orm.py index ee50b172b..f0bc4890a 100644 --- a/src/core/trulens/core/database/orm.py +++ b/src/core/trulens/core/database/orm.py @@ -5,10 +5,12 @@ from sqlite3 import Connection as SQLite3Connection from typing import ClassVar, Dict, Generic, Type, TypeVar -from sqlalchemy import DATETIME +from snowflake.sqlalchemy import OBJECT +from snowflake.sqlalchemy import TIMESTAMP_NTZ from sqlalchemy import VARCHAR from sqlalchemy import Column from sqlalchemy import Engine +from sqlalchemy import Enum from sqlalchemy import Float from sqlalchemy import ForeignKey from sqlalchemy import Text @@ -413,12 +415,7 @@ class Event(base): _table_base_name = "events" - event_id = Column(TYPE_ID, nullable=False, primary_key=True) - """ - The unique identifier for the event. This is just the span_id. - """ - - record = Column(TYPE_JSON, nullable=False) + record = Column(OBJECT, nullable=False) """ For a span, this is an object that includes: - name: the function/procedure that emitted the data @@ -427,36 +424,42 @@ class Event(base): - status: STATUS_CODE_ERROR when the span corresponds to an unhandled exception. Otherwise, STATUS_CODE_UNSET. """ - record_attributes = Column(TYPE_JSON, nullable=False) + record_attributes = Column(OBJECT, nullable=False) """ Attributes of the record that can either come from the user, or based on the TruLens semantic conventions. """ - record_type = Column(VARCHAR(256), nullable=False) + record_type = Column( + Enum(event_schema.EventRecordType), nullable=False + ) """ Specifies the kind of record specified by this row. This will always be "SPAN" for TruLens. """ - resource_attributes = Column(TYPE_JSON, nullable=False) + resource_attributes = Column(OBJECT, nullable=False) """ Reserved. """ - start_timestamp = Column(DATETIME, nullable=False) + start_timestamp = Column( + TIMESTAMP_NTZ, nullable=False, primary_key=True + ) """ The timestamp when the span started. This is a UNIX timestamp in milliseconds. Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. + + Set as the primary key for testing since SQLAlchemy needs one. """ - timestamp = Column(DATETIME, nullable=False) + timestamp = Column(TIMESTAMP_NTZ, nullable=False) """ The timestamp when the span concluded. This is a UNIX timestamp in milliseconds. Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. """ - trace = Column(TYPE_JSON, nullable=False) + trace = Column(OBJECT, nullable=False) """ - Contains the span context, including the trace_id and span_id for the span. + Contains the span context, including the trace_id, parent_id, and span_id for the span. """ @classmethod @@ -465,8 +468,13 @@ def parse( obj: event_schema.Event, redact_keys: bool = False, ) -> ORM.EventTable: + # Attributes stored as objects should be converted to JSON strings, so that + # they can then be parsed via Snowflake's PARSE_JSON function. This is required + # because Snowflake SQLAlchemy doesn't support natively inserting Python dicts + # as objects. + # + # See patch_insert in trulens.core.database.sqlalchemy to learn more. return cls( - event_id=obj.event_id, record=json_utils.json_str_of_obj( obj.record, redact_keys=redact_keys ), diff --git a/src/core/trulens/core/schema/event.py b/src/core/trulens/core/schema/event.py index 5a21f326f..8f4fbd36c 100644 --- a/src/core/trulens/core/schema/event.py +++ b/src/core/trulens/core/schema/event.py @@ -3,23 +3,33 @@ from __future__ import annotations from datetime import datetime +import enum import logging from typing import Any, Dict, Hashable -from trulens.core.schema import types as types_schema from trulens.core.utils import serial as serial_utils +from typing_extensions import TypedDict logger = logging.getLogger(__name__) +class EventRecordType(enum.Enum): + """The enumeration of the possible record types for an event.""" + + SPAN = "SPAN" + + +class Trace(TypedDict): + """The type hint for a trace dictionary.""" + + trace_id: str + parent_id: str + span_id: str + + class Event(serial_utils.SerialModel, Hashable): """The class that represents a single event data entry.""" - event_id: types_schema.EventID # str - """ - The unique identifier for the event. This is just the span_id. - """ - record: Dict[str, Any] """ For a span, this is an object that includes: @@ -34,7 +44,7 @@ class Event(serial_utils.SerialModel, Hashable): Attributes of the record that can either come from the user, or based on the TruLens semantic conventions. """ - record_type: str + record_type: EventRecordType """ Specifies the kind of record specified by this row. This will always be "SPAN" for TruLens. """ @@ -56,15 +66,16 @@ class Event(serial_utils.SerialModel, Hashable): Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. """ - trace: Dict[str, Any] + trace: Trace + """ + The trace context information for the span. + """ def __init__( self, - event_id: types_schema.EventID, **kwargs, ): - super().__init__(event_id=event_id, **kwargs) - self.event_id = event_id + super().__init__(**kwargs) def __hash__(self): - return hash(self.event_id) + return self.trace["span_id"] From 2b03079b531d50fb19559e8e066c1680857cb04d Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:01:38 -0500 Subject: [PATCH 05/59] touchups --- .../database/migrations/versions/10_create_event_table.py | 3 +++ src/core/trulens/core/schema/types.py | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index 3fea460ad..a5429d078 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -26,6 +26,9 @@ def upgrade(config) -> None: # ### commands auto generated by Alembic - please adjust! ### if op.get_context().dialect.name == SnowflakeDialect.name: + # Note: This particular migration is temporary, intended for helping us test + # OTEL integration, hence we only perform this table creation if it's a Snowflake + # table. op.create_table( prefix + "events", sa.Column("record", OBJECT(), nullable=False), diff --git a/src/core/trulens/core/schema/types.py b/src/core/trulens/core/schema/types.py index 5384bb34f..18428c715 100644 --- a/src/core/trulens/core/schema/types.py +++ b/src/core/trulens/core/schema/types.py @@ -83,7 +83,3 @@ def new_call_id() -> CallID: By default these are hashes of dataset content as json. """ - -EventID: TypeAlias = str -"""Unique identifier for a event. -""" From a8e4955663aa96349ba1beae94fb57fcb919ac5d Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:06:21 -0500 Subject: [PATCH 06/59] add event_id --- src/core/trulens/core/database/orm.py | 13 ++++++++----- src/core/trulens/core/schema/event.py | 5 +++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/core/trulens/core/database/orm.py b/src/core/trulens/core/database/orm.py index f0bc4890a..f60ecf79b 100644 --- a/src/core/trulens/core/database/orm.py +++ b/src/core/trulens/core/database/orm.py @@ -424,6 +424,12 @@ class Event(base): - status: STATUS_CODE_ERROR when the span corresponds to an unhandled exception. Otherwise, STATUS_CODE_UNSET. """ + event_id = Column(TYPE_ID, nullable=False, primary_key=True) + """ + Used as primary key for the schema. Not technically present in the event table schema, + but having it here makes it easier to work with the ORM. + """ + record_attributes = Column(OBJECT, nullable=False) """ Attributes of the record that can either come from the user, or based on the TruLens semantic conventions. @@ -441,14 +447,10 @@ class Event(base): Reserved. """ - start_timestamp = Column( - TIMESTAMP_NTZ, nullable=False, primary_key=True - ) + start_timestamp = Column(TIMESTAMP_NTZ, nullable=False) """ The timestamp when the span started. This is a UNIX timestamp in milliseconds. Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. - - Set as the primary key for testing since SQLAlchemy needs one. """ timestamp = Column(TIMESTAMP_NTZ, nullable=False) @@ -475,6 +477,7 @@ def parse( # # See patch_insert in trulens.core.database.sqlalchemy to learn more. return cls( + event_id=obj.event_id, record=json_utils.json_str_of_obj( obj.record, redact_keys=redact_keys ), diff --git a/src/core/trulens/core/schema/event.py b/src/core/trulens/core/schema/event.py index 8f4fbd36c..63fc96e5a 100644 --- a/src/core/trulens/core/schema/event.py +++ b/src/core/trulens/core/schema/event.py @@ -30,6 +30,11 @@ class Trace(TypedDict): class Event(serial_utils.SerialModel, Hashable): """The class that represents a single event data entry.""" + event_id: str + """ + The unique identifier for the event. + """ + record: Dict[str, Any] """ For a span, this is an object that includes: From a6c0b633b1ea486dcf63e3405d31f07508bf4227 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:12:49 -0500 Subject: [PATCH 07/59] add type schema --- src/core/trulens/core/schema/types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/trulens/core/schema/types.py b/src/core/trulens/core/schema/types.py index 18428c715..5384bb34f 100644 --- a/src/core/trulens/core/schema/types.py +++ b/src/core/trulens/core/schema/types.py @@ -83,3 +83,7 @@ def new_call_id() -> CallID: By default these are hashes of dataset content as json. """ + +EventID: TypeAlias = str +"""Unique identifier for a event. +""" From 0ca7ebc9d684b45dd5d7ebe851c555ccbfa8b66d Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:39:24 -0500 Subject: [PATCH 08/59] add event_id --- .../core/database/migrations/versions/10_create_event_table.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index a5429d078..304754c70 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -31,6 +31,7 @@ def upgrade(config) -> None: # table. op.create_table( prefix + "events", + sa.Column("event_id", sa.VARCHAR(length=256), nullable=False), sa.Column("record", OBJECT(), nullable=False), sa.Column("record_attributes", OBJECT(), nullable=False), sa.Column("record_type", sa.VARCHAR(length=256), nullable=False), @@ -38,6 +39,7 @@ def upgrade(config) -> None: sa.Column("start_timestamp", TIMESTAMP_NTZ(), nullable=False), sa.Column("timestamp", TIMESTAMP_NTZ(), nullable=False), sa.Column("trace", OBJECT(), nullable=False), + sa.PrimaryKeyConstraint("event_id"), ) # ### end Alembic commands ### From 1c6cbc236433196669cc145387760c2ae2707607 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:47:39 -0500 Subject: [PATCH 09/59] minor updates --- .../versions/10_create_event_table.py | 5 +- .../static/golden/api.trulens_eval.3.11.yaml | 444 +++++++++--------- 2 files changed, 226 insertions(+), 223 deletions(-) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index 304754c70..ea64763b7 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -26,9 +26,12 @@ def upgrade(config) -> None: # ### commands auto generated by Alembic - please adjust! ### if op.get_context().dialect.name == SnowflakeDialect.name: - # Note: This particular migration is temporary, intended for helping us test + # Note: + # 1. This particular migration is temporary, intended for helping us test # OTEL integration, hence we only perform this table creation if it's a Snowflake # table. + # 2. Event tables technically don't have event_id, but including it as a column here + # because SQL alchemy needs one, and there aren't really suitable columns. op.create_table( prefix + "events", sa.Column("event_id", sa.VARCHAR(length=256), nullable=False), diff --git a/tests/unit/static/golden/api.trulens_eval.3.11.yaml b/tests/unit/static/golden/api.trulens_eval.3.11.yaml index a0fd878e3..e863fd0a7 100644 --- a/tests/unit/static/golden/api.trulens_eval.3.11.yaml +++ b/tests/unit/static/golden/api.trulens_eval.3.11.yaml @@ -78,7 +78,7 @@ trulens_eval.AzureOpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -86,7 +86,7 @@ trulens_eval.AzureOpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -170,7 +170,7 @@ trulens_eval.Bedrock: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -178,7 +178,7 @@ trulens_eval.Bedrock: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_id: builtins.str model_json_schema: builtins.classmethod @@ -256,7 +256,7 @@ trulens_eval.Cortex: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -264,7 +264,7 @@ trulens_eval.Cortex: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -326,14 +326,14 @@ trulens_eval.Feedback: builtins.NoneType] json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -400,14 +400,14 @@ trulens_eval.Huggingface: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -450,14 +450,14 @@ trulens_eval.HuggingfaceLocal: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -526,7 +526,7 @@ trulens_eval.Langchain: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -534,7 +534,7 @@ trulens_eval.Langchain: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -617,7 +617,7 @@ trulens_eval.LiteLLM: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -625,7 +625,7 @@ trulens_eval.LiteLLM: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -700,7 +700,7 @@ trulens_eval.OpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -708,7 +708,7 @@ trulens_eval.OpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -762,14 +762,14 @@ trulens_eval.Provider: get_class: builtins.staticmethod json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -862,14 +862,14 @@ trulens_eval.Tru: get_records_and_feedback: builtins.function json: builtins.function migrate_database: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -940,14 +940,14 @@ trulens_eval.TruBasicApp: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1039,14 +1039,14 @@ trulens_eval.TruChain: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1139,14 +1139,14 @@ trulens_eval.TruCustomApp: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1236,14 +1236,14 @@ trulens_eval.TruLlama: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1334,14 +1334,14 @@ trulens_eval.TruRails: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1433,14 +1433,14 @@ trulens_eval.TruVirtual: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1563,14 +1563,14 @@ trulens_eval.app.App: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1823,14 +1823,14 @@ trulens_eval.database.base.DB: insert_record: builtins.function json: builtins.function migrate_database: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -1980,14 +1980,14 @@ trulens_eval.database.sqlalchemy.SQLAlchemyDB: insert_record: builtins.function json: builtins.function migrate_database: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2092,7 +2092,7 @@ trulens_eval.feedback.AzureOpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -2100,7 +2100,7 @@ trulens_eval.feedback.AzureOpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2184,7 +2184,7 @@ trulens_eval.feedback.Bedrock: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -2192,7 +2192,7 @@ trulens_eval.feedback.Bedrock: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_id: builtins.str model_json_schema: builtins.classmethod @@ -2270,7 +2270,7 @@ trulens_eval.feedback.Cortex: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -2278,7 +2278,7 @@ trulens_eval.feedback.Cortex: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2326,14 +2326,14 @@ trulens_eval.feedback.Embeddings: json: builtins.function load: builtins.staticmethod manhattan_distance: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2385,14 +2385,14 @@ trulens_eval.feedback.Feedback: builtins.NoneType] json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2453,14 +2453,14 @@ trulens_eval.feedback.GroundTruthAgreement: json: builtins.function load: builtins.staticmethod mae: builtins.property - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2505,14 +2505,14 @@ trulens_eval.feedback.Huggingface: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2555,14 +2555,14 @@ trulens_eval.feedback.HuggingfaceLocal: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2631,7 +2631,7 @@ trulens_eval.feedback.Langchain: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -2639,7 +2639,7 @@ trulens_eval.feedback.Langchain: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2715,7 +2715,7 @@ trulens_eval.feedback.LiteLLM: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -2723,7 +2723,7 @@ trulens_eval.feedback.LiteLLM: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2798,7 +2798,7 @@ trulens_eval.feedback.OpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -2806,7 +2806,7 @@ trulens_eval.feedback.OpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2868,14 +2868,14 @@ trulens_eval.feedback.embeddings.Embeddings: json: builtins.function load: builtins.staticmethod manhattan_distance: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -2936,14 +2936,14 @@ trulens_eval.feedback.feedback.Feedback: builtins.NoneType] json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3020,14 +3020,14 @@ trulens_eval.feedback.groundtruth.GroundTruthAgreement: json: builtins.function load: builtins.staticmethod mae: builtins.property - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3158,7 +3158,7 @@ trulens_eval.feedback.provider.AzureOpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3166,7 +3166,7 @@ trulens_eval.feedback.provider.AzureOpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3250,7 +3250,7 @@ trulens_eval.feedback.provider.Bedrock: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3258,7 +3258,7 @@ trulens_eval.feedback.provider.Bedrock: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_id: builtins.str model_json_schema: builtins.classmethod @@ -3336,7 +3336,7 @@ trulens_eval.feedback.provider.Cortex: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3344,7 +3344,7 @@ trulens_eval.feedback.provider.Cortex: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3393,14 +3393,14 @@ trulens_eval.feedback.provider.Huggingface: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3443,14 +3443,14 @@ trulens_eval.feedback.provider.HuggingfaceLocal: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3519,7 +3519,7 @@ trulens_eval.feedback.provider.Langchain: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3527,7 +3527,7 @@ trulens_eval.feedback.provider.Langchain: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3603,7 +3603,7 @@ trulens_eval.feedback.provider.LiteLLM: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3611,7 +3611,7 @@ trulens_eval.feedback.provider.LiteLLM: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3686,7 +3686,7 @@ trulens_eval.feedback.provider.OpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3694,7 +3694,7 @@ trulens_eval.feedback.provider.OpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3748,14 +3748,14 @@ trulens_eval.feedback.provider.Provider: get_class: builtins.staticmethod json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3827,7 +3827,7 @@ trulens_eval.feedback.provider.base.LLMProvider: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3835,7 +3835,7 @@ trulens_eval.feedback.provider.base.LLMProvider: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3880,14 +3880,14 @@ trulens_eval.feedback.provider.base.Provider: get_class: builtins.staticmethod json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -3959,7 +3959,7 @@ trulens_eval.feedback.provider.bedrock.Bedrock: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -3967,7 +3967,7 @@ trulens_eval.feedback.provider.bedrock.Bedrock: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_id: builtins.str model_json_schema: builtins.classmethod @@ -4051,7 +4051,7 @@ trulens_eval.feedback.provider.cortex.Cortex: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -4059,7 +4059,7 @@ trulens_eval.feedback.provider.cortex.Cortex: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4132,14 +4132,14 @@ trulens_eval.feedback.provider.endpoint.BedrockEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4198,14 +4198,14 @@ trulens_eval.feedback.provider.endpoint.CortexEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4273,14 +4273,14 @@ trulens_eval.feedback.provider.endpoint.DummyEndpoint: load: builtins.staticmethod loading_prob: builtins.property loading_time: builtins.property - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4343,14 +4343,14 @@ trulens_eval.feedback.provider.endpoint.Endpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4411,14 +4411,14 @@ trulens_eval.feedback.provider.endpoint.HuggingfaceEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4479,14 +4479,14 @@ trulens_eval.feedback.provider.endpoint.LangchainEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4545,14 +4545,14 @@ trulens_eval.feedback.provider.endpoint.LiteLLMEndpoint: json: builtins.function litellm_provider: builtins.str load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4601,14 +4601,14 @@ trulens_eval.feedback.provider.endpoint.OpenAIClient: formatted_objects: _contextvars.ContextVar from_orm: builtins.classmethod json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4651,14 +4651,14 @@ trulens_eval.feedback.provider.endpoint.OpenAIEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4737,14 +4737,14 @@ trulens_eval.feedback.provider.endpoint.base.DummyEndpoint: load: builtins.staticmethod loading_prob: builtins.property loading_time: builtins.property - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4807,14 +4807,14 @@ trulens_eval.feedback.provider.endpoint.base.Endpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4867,14 +4867,14 @@ trulens_eval.feedback.provider.endpoint.base.EndpointCallback: handle_generation: builtins.function handle_generation_chunk: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4918,14 +4918,14 @@ trulens_eval.feedback.provider.endpoint.bedrock.BedrockCallback: handle_generation: builtins.function handle_generation_chunk: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -4968,14 +4968,14 @@ trulens_eval.feedback.provider.endpoint.bedrock.BedrockEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5036,14 +5036,14 @@ trulens_eval.feedback.provider.endpoint.cortex.CortexCallback: handle_generation: builtins.function handle_generation_chunk: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5085,14 +5085,14 @@ trulens_eval.feedback.provider.endpoint.cortex.CortexEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5152,14 +5152,14 @@ trulens_eval.feedback.provider.endpoint.hugs.HuggingfaceCallback: handle_generation: builtins.function handle_generation_chunk: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5204,14 +5204,14 @@ trulens_eval.feedback.provider.endpoint.hugs.HuggingfaceEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5273,14 +5273,14 @@ trulens_eval.feedback.provider.endpoint.langchain.LangchainCallback: handle_generation: builtins.function handle_generation_chunk: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5323,14 +5323,14 @@ trulens_eval.feedback.provider.endpoint.langchain.LangchainEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5390,14 +5390,14 @@ trulens_eval.feedback.provider.endpoint.litellm.LiteLLMCallback: handle_generation: builtins.function handle_generation_chunk: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5440,14 +5440,14 @@ trulens_eval.feedback.provider.endpoint.litellm.LiteLLMEndpoint: json: builtins.function litellm_provider: builtins.str load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5510,14 +5510,14 @@ trulens_eval.feedback.provider.endpoint.openai.OpenAICallback: handle_generation_chunk: builtins.function json: builtins.function langchain_handler: langchain_community.callbacks.openai_info.OpenAICallbackHandler - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5550,14 +5550,14 @@ trulens_eval.feedback.provider.endpoint.openai.OpenAIClient: formatted_objects: _contextvars.ContextVar from_orm: builtins.classmethod json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5600,14 +5600,14 @@ trulens_eval.feedback.provider.endpoint.openai.OpenAIEndpoint: instrumented_methods: collections.defaultdict json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5677,14 +5677,14 @@ trulens_eval.feedback.provider.hugs.Dummy: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5727,14 +5727,14 @@ trulens_eval.feedback.provider.hugs.Huggingface: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5777,14 +5777,14 @@ trulens_eval.feedback.provider.hugs.HuggingfaceBase: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5827,14 +5827,14 @@ trulens_eval.feedback.provider.hugs.HuggingfaceLocal: json: builtins.function language_match: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5909,7 +5909,7 @@ trulens_eval.feedback.provider.langchain.Langchain: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -5917,7 +5917,7 @@ trulens_eval.feedback.provider.langchain.Langchain: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -5999,7 +5999,7 @@ trulens_eval.feedback.provider.litellm.LiteLLM: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -6007,7 +6007,7 @@ trulens_eval.feedback.provider.litellm.LiteLLM: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6090,7 +6090,7 @@ trulens_eval.feedback.provider.openai.AzureOpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -6098,7 +6098,7 @@ trulens_eval.feedback.provider.openai.AzureOpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6182,7 +6182,7 @@ trulens_eval.feedback.provider.openai.OpenAI: misogyny: builtins.function misogyny_with_cot_reasons: builtins.function model_agreement: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function @@ -6190,7 +6190,7 @@ trulens_eval.feedback.provider.openai.OpenAI: model_dump_json: builtins.function model_engine: builtins.str model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6420,14 +6420,14 @@ trulens_eval.schema.app.AppDefinition: jsonify_extra: builtins.function load: builtins.staticmethod metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6530,14 +6530,14 @@ trulens_eval.schema.feedback.FeedbackCall: from_orm: builtins.classmethod json: builtins.function meta: typing.Dict[builtins.str, typing.Any] - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6593,14 +6593,14 @@ trulens_eval.schema.feedback.FeedbackDefinition: builtins.NoneType] json: builtins.function load: builtins.staticmethod - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6666,14 +6666,14 @@ trulens_eval.schema.feedback.FeedbackResult: from_orm: builtins.classmethod json: builtins.function last_ts: datetime.datetime - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6773,14 +6773,14 @@ trulens_eval.schema.record.Record: builtins.NoneType, typing.Sequence[typing.Any], typing.Dict[builtins.str, typing.Any]] meta: typing.Union[builtins.str, builtins.int, builtins.float, builtins.bytes, builtins.NoneType, typing.Sequence[typing.Any], typing.Dict[builtins.str, typing.Any]] - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6819,14 +6819,14 @@ trulens_eval.schema.record.RecordAppCall: from_orm: builtins.classmethod json: builtins.function method: builtins.property - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6863,14 +6863,14 @@ trulens_eval.schema.record.RecordAppCallMethod: from_orm: builtins.classmethod json: builtins.function method: trulens.core.utils.pyschema.Method - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -6960,14 +6960,14 @@ trulens_eval.tru.Tru: get_records_and_feedback: builtins.function json: builtins.function migrate_database: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -7046,14 +7046,14 @@ trulens_eval.tru_basic_app.TruBasicApp: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -7189,14 +7189,14 @@ trulens_eval.tru_chain.TruChain: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -7297,14 +7297,14 @@ trulens_eval.tru_custom_app.TruCustomApp: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -7425,14 +7425,14 @@ trulens_eval.tru_llama.TruLlama: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -7591,14 +7591,14 @@ trulens_eval.tru_rails.TruRails: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -7698,14 +7698,14 @@ trulens_eval.tru_virtual.TruVirtual: manage_pending_feedback_results_thread: typing.Optional[trulens.core.utils.threading.Thread, builtins.NoneType] metadata: typing.Dict - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -7787,14 +7787,14 @@ trulens_eval.tru_virtual.VirtualRecord: builtins.NoneType, typing.Sequence[typing.Any], typing.Dict[builtins.str, typing.Any]] meta: typing.Union[builtins.str, builtins.int, builtins.float, builtins.bytes, builtins.NoneType, typing.Sequence[typing.Any], typing.Dict[builtins.str, typing.Any]] - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8128,14 +8128,14 @@ trulens_eval.utils.pyschema.Bindings: json: builtins.function kwargs: typing.Dict[builtins.str, typing.Any] load: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8168,14 +8168,14 @@ trulens_eval.utils.pyschema.Class: from_orm: builtins.classmethod json: builtins.function load: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8212,14 +8212,14 @@ trulens_eval.utils.pyschema.Function: from_orm: builtins.classmethod json: builtins.function load: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8253,14 +8253,14 @@ trulens_eval.utils.pyschema.FunctionOrMethod: from_orm: builtins.classmethod json: builtins.function load: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8291,14 +8291,14 @@ trulens_eval.utils.pyschema.Method: from_orm: builtins.classmethod json: builtins.function load: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8332,14 +8332,14 @@ trulens_eval.utils.pyschema.Module: from_orm: builtins.classmethod json: builtins.function load: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8376,14 +8376,14 @@ trulens_eval.utils.pyschema.Obj: init_bindings: typing.Optional[trulens.core.utils.pyschema.Bindings, builtins.NoneType] json: builtins.function load: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8536,14 +8536,14 @@ trulens_eval.utils.serial.Collect: get: builtins.function get_sole_item: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8574,14 +8574,14 @@ trulens_eval.utils.serial.GetAttribute: get_item_or_attribute: builtins.function get_sole_item: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8611,14 +8611,14 @@ trulens_eval.utils.serial.GetIndex: get_sole_item: builtins.function index: builtins.int json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8648,14 +8648,14 @@ trulens_eval.utils.serial.GetIndices: get_sole_item: builtins.function indices: typing.Tuple[builtins.int, ...] json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8686,14 +8686,14 @@ trulens_eval.utils.serial.GetItem: get_sole_item: builtins.function item: builtins.str json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8724,14 +8724,14 @@ trulens_eval.utils.serial.GetItemOrAttribute: get_sole_item: builtins.function item_or_attribute: builtins.str json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8761,14 +8761,14 @@ trulens_eval.utils.serial.GetItems: get_sole_item: builtins.function items: typing.Tuple[builtins.str, ...] json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8797,14 +8797,14 @@ trulens_eval.utils.serial.GetSlice: get: builtins.function get_sole_item: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod @@ -8886,14 +8886,14 @@ trulens_eval.utils.serial.StepItemOrAttribute: get_item_or_attribute: builtins.function get_sole_item: builtins.function json: builtins.function - model_computed_fields: builtins.dict + model_computed_fields: builtins.property model_config: builtins.dict model_construct: builtins.classmethod model_copy: builtins.function model_dump: builtins.function model_dump_json: builtins.function model_extra: builtins.property - model_fields: builtins.dict + model_fields: builtins.property model_fields_set: builtins.property model_json_schema: builtins.classmethod model_parametrized_name: builtins.classmethod From 3059d799f8593c876cf5e0739e3a7c41079a1f46 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 19 Dec 2024 08:56:42 -0500 Subject: [PATCH 10/59] pr feedback --- .../database/migrations/versions/10_create_event_table.py | 4 ---- src/core/trulens/core/database/orm.py | 5 +++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index ea64763b7..553837ee5 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -24,7 +24,6 @@ def upgrade(config) -> None: if prefix is None: raise RuntimeError("trulens.table_prefix is not set") - # ### commands auto generated by Alembic - please adjust! ### if op.get_context().dialect.name == SnowflakeDialect.name: # Note: # 1. This particular migration is temporary, intended for helping us test @@ -44,7 +43,6 @@ def upgrade(config) -> None: sa.Column("trace", OBJECT(), nullable=False), sa.PrimaryKeyConstraint("event_id"), ) - # ### end Alembic commands ### def downgrade(config) -> None: @@ -53,7 +51,5 @@ def downgrade(config) -> None: if prefix is None: raise RuntimeError("trulens.table_prefix is not set") - # ### commands auto generated by Alembic - please adjust! ### if op.get_context().dialect.name == SnowflakeDialect.name: op.drop_table(prefix + "events") - # ### end Alembic commands ### diff --git a/src/core/trulens/core/database/orm.py b/src/core/trulens/core/database/orm.py index f60ecf79b..b4db87336 100644 --- a/src/core/trulens/core/database/orm.py +++ b/src/core/trulens/core/database/orm.py @@ -5,8 +5,6 @@ from sqlite3 import Connection as SQLite3Connection from typing import ClassVar, Dict, Generic, Type, TypeVar -from snowflake.sqlalchemy import OBJECT -from snowflake.sqlalchemy import TIMESTAMP_NTZ from sqlalchemy import VARCHAR from sqlalchemy import Column from sqlalchemy import Engine @@ -407,6 +405,9 @@ def parse( ) class Event(base): + from snowflake.sqlalchemy import OBJECT + from snowflake.sqlalchemy import TIMESTAMP_NTZ + """ ORM class for OTEL traces/spans. This should be kept in line with the event table https://docs.snowflake.com/en/developer-guide/logging-tracing/event-table-columns#data-for-trace-events From 021a43cd1003c8605354b480a33cfe67855e0600 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 19 Dec 2024 15:21:59 -0500 Subject: [PATCH 11/59] update --- .../versions/10_create_event_table.py | 37 +++++++------------ src/core/trulens/core/database/orm.py | 23 ++++-------- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py index 553837ee5..3dfe4eae6 100644 --- a/src/core/trulens/core/database/migrations/versions/10_create_event_table.py +++ b/src/core/trulens/core/database/migrations/versions/10_create_event_table.py @@ -6,9 +6,6 @@ """ from alembic import op -from snowflake.sqlalchemy import OBJECT -from snowflake.sqlalchemy import TIMESTAMP_NTZ -from snowflake.sqlalchemy import dialect as SnowflakeDialect import sqlalchemy as sa # revision identifiers, used by Alembic. @@ -24,25 +21,18 @@ def upgrade(config) -> None: if prefix is None: raise RuntimeError("trulens.table_prefix is not set") - if op.get_context().dialect.name == SnowflakeDialect.name: - # Note: - # 1. This particular migration is temporary, intended for helping us test - # OTEL integration, hence we only perform this table creation if it's a Snowflake - # table. - # 2. Event tables technically don't have event_id, but including it as a column here - # because SQL alchemy needs one, and there aren't really suitable columns. - op.create_table( - prefix + "events", - sa.Column("event_id", sa.VARCHAR(length=256), nullable=False), - sa.Column("record", OBJECT(), nullable=False), - sa.Column("record_attributes", OBJECT(), nullable=False), - sa.Column("record_type", sa.VARCHAR(length=256), nullable=False), - sa.Column("resource_attributes", OBJECT(), nullable=False), - sa.Column("start_timestamp", TIMESTAMP_NTZ(), nullable=False), - sa.Column("timestamp", TIMESTAMP_NTZ(), nullable=False), - sa.Column("trace", OBJECT(), nullable=False), - sa.PrimaryKeyConstraint("event_id"), - ) + op.create_table( + prefix + "events", + sa.Column("event_id", sa.VARCHAR(length=256), nullable=False), + sa.Column("record", sa.JSON(), nullable=False), + sa.Column("record_attributes", sa.JSON(), nullable=False), + sa.Column("record_type", sa.VARCHAR(length=256), nullable=False), + sa.Column("resource_attributes", sa.JSON(), nullable=False), + sa.Column("start_timestamp", sa.TIMESTAMP(), nullable=False), + sa.Column("timestamp", sa.TIMESTAMP(), nullable=False), + sa.Column("trace", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("event_id"), + ) def downgrade(config) -> None: @@ -51,5 +41,4 @@ def downgrade(config) -> None: if prefix is None: raise RuntimeError("trulens.table_prefix is not set") - if op.get_context().dialect.name == SnowflakeDialect.name: - op.drop_table(prefix + "events") + op.drop_table(prefix + "events") diff --git a/src/core/trulens/core/database/orm.py b/src/core/trulens/core/database/orm.py index b4db87336..8db63c06f 100644 --- a/src/core/trulens/core/database/orm.py +++ b/src/core/trulens/core/database/orm.py @@ -5,6 +5,8 @@ from sqlite3 import Connection as SQLite3Connection from typing import ClassVar, Dict, Generic, Type, TypeVar +from sqlalchemy import JSON +from sqlalchemy import TIMESTAMP from sqlalchemy import VARCHAR from sqlalchemy import Column from sqlalchemy import Engine @@ -405,9 +407,6 @@ def parse( ) class Event(base): - from snowflake.sqlalchemy import OBJECT - from snowflake.sqlalchemy import TIMESTAMP_NTZ - """ ORM class for OTEL traces/spans. This should be kept in line with the event table https://docs.snowflake.com/en/developer-guide/logging-tracing/event-table-columns#data-for-trace-events @@ -416,7 +415,7 @@ class Event(base): _table_base_name = "events" - record = Column(OBJECT, nullable=False) + record = Column(JSON, nullable=False) """ For a span, this is an object that includes: - name: the function/procedure that emitted the data @@ -431,7 +430,7 @@ class Event(base): but having it here makes it easier to work with the ORM. """ - record_attributes = Column(OBJECT, nullable=False) + record_attributes = Column(JSON, nullable=False) """ Attributes of the record that can either come from the user, or based on the TruLens semantic conventions. """ @@ -443,24 +442,24 @@ class Event(base): Specifies the kind of record specified by this row. This will always be "SPAN" for TruLens. """ - resource_attributes = Column(OBJECT, nullable=False) + resource_attributes = Column(JSON, nullable=False) """ Reserved. """ - start_timestamp = Column(TIMESTAMP_NTZ, nullable=False) + start_timestamp = Column(TIMESTAMP, nullable=False) """ The timestamp when the span started. This is a UNIX timestamp in milliseconds. Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. """ - timestamp = Column(TIMESTAMP_NTZ, nullable=False) + timestamp = Column(TIMESTAMP, nullable=False) """ The timestamp when the span concluded. This is a UNIX timestamp in milliseconds. Note: The Snowflake event table uses the TIMESTAMP_NTZ data type for this column. """ - trace = Column(OBJECT, nullable=False) + trace = Column(JSON, nullable=False) """ Contains the span context, including the trace_id, parent_id, and span_id for the span. """ @@ -471,12 +470,6 @@ def parse( obj: event_schema.Event, redact_keys: bool = False, ) -> ORM.EventTable: - # Attributes stored as objects should be converted to JSON strings, so that - # they can then be parsed via Snowflake's PARSE_JSON function. This is required - # because Snowflake SQLAlchemy doesn't support natively inserting Python dicts - # as objects. - # - # See patch_insert in trulens.core.database.sqlalchemy to learn more. return cls( event_id=obj.event_id, record=json_utils.json_str_of_obj( From 42f0575e499a4f1b1eb4eada85d04d065402f579 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 18:05:54 -0500 Subject: [PATCH 12/59] ORM update --- examples/experimental/otel_exporter.ipynb | 5 +---- src/core/trulens/core/database/orm.py | 16 ++++------------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 4b6c16a96..0eddcb41c 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -117,10 +117,7 @@ "source": [ "from trulens.core.session import TruSession\n", "\n", - "SERVICE_NAME = \"trulens\"\n", - "\n", - "\n", - "# session = TruSession()" + "SERVICE_NAME = \"trulens\"" ] }, { diff --git a/src/core/trulens/core/database/orm.py b/src/core/trulens/core/database/orm.py index 8db63c06f..bc8387f55 100644 --- a/src/core/trulens/core/database/orm.py +++ b/src/core/trulens/core/database/orm.py @@ -472,21 +472,13 @@ def parse( ) -> ORM.EventTable: return cls( event_id=obj.event_id, - record=json_utils.json_str_of_obj( - obj.record, redact_keys=redact_keys - ), - record_attributes=json_utils.json_str_of_obj( - obj.record_attributes, redact_keys=redact_keys - ), + record=obj.record, + record_attributes=obj.record_attributes, record_type=obj.record_type, - resource_attributes=json_utils.json_str_of_obj( - obj.resource_attributes, redact_keys=redact_keys - ), + resource_attributes=obj.resource_attributes, start_timestamp=obj.start_timestamp, timestamp=obj.timestamp, - trace=json_utils.json_str_of_obj( - obj.trace, redact_keys=redact_keys - ), + trace=obj.trace, ) configure_mappers() # IMPORTANT From 9c313b46566b04bcc42a5e1b0fd9c9385bab04b6 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 11 Dec 2024 15:36:32 -0800 Subject: [PATCH 13/59] save --- examples/experimental/otel_exporter.ipynb | 276 ++++++++---------- src/core/trulens/core/database/base.py | 13 + .../trulens/core/database/connector/base.py | 19 ++ src/core/trulens/core/database/sqlalchemy.py | 11 + .../otel_tracing/core/exporter.py | 47 +++ .../experimental/otel_tracing/core/init.py | 38 +++ 6 files changed, 254 insertions(+), 150 deletions(-) create mode 100644 src/core/trulens/experimental/otel_tracing/core/exporter.py create mode 100644 src/core/trulens/experimental/otel_tracing/core/init.py diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 0eddcb41c..fb00a0180 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -1,153 +1,129 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install opentelemetry-api\n", - "# !pip install opentelemetry-sdk" - ] + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install opentelemetry-api\n", + "# !pip install opentelemetry-sdk" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Add base dir to path to be able to access test folder.\n", + "base_dir = Path().cwd().parent.parent.resolve()\n", + "if str(base_dir) not in sys.path:\n", + " print(f\"Adding {base_dir} to sys.path\")\n", + " sys.path.append(str(base_dir))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Callable\n", + "\n", + "from opentelemetry import trace\n", + "from trulens.apps.custom import instrument\n", + "from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME\n", + "\n", + "\n", + "def decorator(func: Callable):\n", + " tracer = trace.get_tracer(TRULENS_SERVICE_NAME)\n", + "\n", + " def wrapper(*args, **kwargs):\n", + " print(\"start wrap\")\n", + "\n", + " with tracer.start_as_current_span(\"custom\"):\n", + " result = func(*args, **kwargs)\n", + " span = trace.get_current_span()\n", + " print(\"---span---\")\n", + " print(span.get_span_context())\n", + " span.set_attribute(\"result\", result)\n", + " span.set_status(trace.Status(trace.StatusCode.OK))\n", + " return result\n", + "\n", + " return wrapper" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from examples.dev.dummy_app.dummy import Dummy\n", + "\n", + "\n", + "class TestApp(Dummy):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " @decorator\n", + " @instrument\n", + " def respond_to_query(self, query: str) -> str:\n", + " return f\"answer: {self.nested(query)}\"\n", + "\n", + " @decorator\n", + " @instrument\n", + " def nested(self, query: str) -> str:\n", + " return f\"nested: {query}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.core.session import TruSession\n", + "from trulens.experimental.otel_tracing.core.init import init\n", + "\n", + "session = TruSession()\n", + "init(session)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_app = TestApp()\n", + "\n", + "test_app.respond_to_query(\"test\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "trulens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import sys\n", - "\n", - "# Add base dir to path to be able to access test folder.\n", - "base_dir = Path().cwd().parent.parent.resolve()\n", - "if str(base_dir) not in sys.path:\n", - " print(f\"Adding {base_dir} to sys.path\")\n", - " sys.path.append(str(base_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from opentelemetry import trace\n", - "from opentelemetry.sdk.resources import Resource\n", - "from opentelemetry.sdk.trace import TracerProvider\n", - "from opentelemetry.sdk.trace.export import BatchSpanProcessor\n", - "from opentelemetry.sdk.trace.export import ConsoleSpanExporter\n", - "\n", - "SERVICE_NAME = \"trulens\"\n", - "\n", - "\n", - "def init():\n", - " # Use Resource.create() instead of constructor directly\n", - " resource = Resource.create({\"service.name\": SERVICE_NAME})\n", - "\n", - " trace.set_tracer_provider(TracerProvider(resource=resource))\n", - " trace.get_tracer_provider().add_span_processor(\n", - " BatchSpanProcessor(ConsoleSpanExporter())\n", - " )\n", - "\n", - "\n", - "init()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from typing import Callable\n", - "\n", - "from trulens.apps.custom import instrument\n", - "\n", - "SERVICE_NAME = \"trulens\"\n", - "\n", - "\n", - "def decorator(func: Callable):\n", - " tracer = trace.get_tracer(SERVICE_NAME)\n", - "\n", - " def wrapper(*args, **kwargs):\n", - " print(\"start wrap\")\n", - "\n", - " with tracer.start_as_current_span(\"custom\"):\n", - " result = func(*args, **kwargs)\n", - " span = trace.get_current_span()\n", - " print(\"---span---\")\n", - " print(span.get_span_context())\n", - " span.set_attribute(\"result\", result)\n", - " span.set_status(trace.Status(trace.StatusCode.OK))\n", - " return result\n", - "\n", - " return wrapper" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from examples.dev.dummy_app.dummy import Dummy\n", - "\n", - "\n", - "class TestApp(Dummy):\n", - " def __init__(self):\n", - " super().__init__()\n", - "\n", - " @decorator\n", - " @instrument\n", - " def respond_to_query(self, query: str) -> str:\n", - " return f\"answer: {self.nested(query)}\"\n", - "\n", - " @decorator\n", - " @instrument\n", - " def nested(self, query: str) -> str:\n", - " return f\"nested: {query}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.core.session import TruSession\n", - "\n", - "SERVICE_NAME = \"trulens\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "session = TruSession()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "trulens", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/src/core/trulens/core/database/base.py b/src/core/trulens/core/database/base.py index 6999baf1f..36b76b006 100644 --- a/src/core/trulens/core/database/base.py +++ b/src/core/trulens/core/database/base.py @@ -6,6 +6,7 @@ import pandas as pd from trulens.core.schema import app as app_schema from trulens.core.schema import dataset as dataset_schema +from trulens.core.schema import event as event_schema from trulens.core.schema import feedback as feedback_schema from trulens.core.schema import groundtruth as groundtruth_schema from trulens.core.schema import record as record_schema @@ -432,3 +433,15 @@ def get_datasets(self) -> pd.DataFrame: A dataframe with the datasets. """ raise NotImplementedError() + + @abc.abstractmethod + def insert_event(self, event: event_schema.Event) -> types_schema.EventID: + """Insert an event into the database. + + Args: + event: The event to insert. + + Returns: + The id of the given event. + """ + raise NotImplementedError() diff --git a/src/core/trulens/core/database/connector/base.py b/src/core/trulens/core/database/connector/base.py index f6cd14644..3a181c045 100644 --- a/src/core/trulens/core/database/connector/base.py +++ b/src/core/trulens/core/database/connector/base.py @@ -20,6 +20,7 @@ from trulens.core._utils.pycompat import Future # code style exception from trulens.core.database import base as core_db from trulens.core.schema import app as app_schema +from trulens.core.schema import event as event_schema from trulens.core.schema import feedback as feedback_schema from trulens.core.schema import record as record_schema from trulens.core.schema import types as types_schema @@ -408,3 +409,21 @@ def get_leaderboard( .mean() .sort_values(by=feedback_cols, ascending=False) ) + + def add_event(self, event: event_schema.Event): + """ + Add an event to the database. + + Args: + event: The event to add to the database. + """ + return self.db.insert_event(event=event) + + def add_events(self, events: List[event_schema.Event]): + """ + Add multiple events to the database. + + Args: + events: A list of events to add to the database. + """ + return [self.add_event(event=event) for event in events] diff --git a/src/core/trulens/core/database/sqlalchemy.py b/src/core/trulens/core/database/sqlalchemy.py index 48ced3587..0aca9c3c0 100644 --- a/src/core/trulens/core/database/sqlalchemy.py +++ b/src/core/trulens/core/database/sqlalchemy.py @@ -44,6 +44,7 @@ from trulens.core.schema import groundtruth as groundtruth_schema from trulens.core.schema import record as record_schema from trulens.core.schema import types as types_schema +from trulens.core.schema.event import Event from trulens.core.utils import pyschema as pyschema_utils from trulens.core.utils import python as python_utils from trulens.core.utils import serial as serial_utils @@ -973,6 +974,16 @@ def get_datasets(self) -> pd.DataFrame: columns=["dataset_id", "name", "meta"], ) + def insert_event(self, event: Event) -> types_schema.EventID: + """See [DB.insert_event][trulens.core.database.base.DB.insert_event].""" + with self.session.begin() as session: + _event = self.orm.Event.parse(event, redact_keys=self.redact_keys) + session.merge(_event) + logger.info( + f"{text_utils.UNICODE_CHECK} added event {_event.event_id}" + ) + return _event.event_id + # Use this Perf for missing Perfs. # TODO: Migrate the database instead. diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py new file mode 100644 index 000000000..558767cc9 --- /dev/null +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -0,0 +1,47 @@ +from typing import Sequence + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter +from opentelemetry.sdk.trace.export import SpanExportResult +from trulens.core.database import connector as core_connector +from trulens.core.schema import event as event_schema + + +class TruLensDBSpanExporter(SpanExporter): + """ + Implementation of :class:`SpanExporter` that flushes the spans to the database in the TruLens session. + """ + + connector: core_connector.DBConnector + + def __init__(self, connector: core_connector.DBConnector): + self.connector = connector + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + for span in spans: + context = span.get_span_context() + + if context is None: + continue + + event = event_schema.Event( + event_id=str(context.span_id), + record=span.attributes, + record_attributes=span.attributes, + record_type="span", + resource_attributes=span.resource.attributes, + start_timestamp=span.start_time, + timestamp=span.end_time, + trace={ + "trace_id": str(context.trace_id), + "parent_id": str(context.span_id), + }, + ) + self.connector.add_event(event) + print(event) + return SpanExportResult.SUCCESS + + """Immediately export all spans""" + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True diff --git a/src/core/trulens/experimental/otel_tracing/core/init.py b/src/core/trulens/experimental/otel_tracing/core/init.py new file mode 100644 index 000000000..9f9a719ac --- /dev/null +++ b/src/core/trulens/experimental/otel_tracing/core/init.py @@ -0,0 +1,38 @@ +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.export import ConsoleSpanExporter +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from trulens.core.session import TruSession +from trulens.experimental.otel_tracing.core.exporter import ( + TruLensDBSpanExporter, +) + +TRULENS_SERVICE_NAME = "trulens" + + +def init(session: TruSession, debug: bool = False): + """Initialize the OpenTelemetry SDK with TruLens configuration.""" + resource = Resource.create({"service.name": TRULENS_SERVICE_NAME}) + provider = TracerProvider(resource=resource) + trace.set_tracer_provider(provider) + + if debug: + print( + "Initializing OpenTelemetry with TruLens configuration for console debugging" + ) + # Add a console exporter for debugging purposes + console_exporter = ConsoleSpanExporter() + console_processor = SimpleSpanProcessor(console_exporter) + provider.add_span_processor(console_processor) + + if session.connector: + print("Exporting traces to the TruLens database") + + # TODO: Validate that the table exists. + + # Add the TruLens database exporter + db_exporter = TruLensDBSpanExporter(session.connector) + db_processor = BatchSpanProcessor(db_exporter) + provider.add_span_processor(db_processor) From 20bb651cd2cf9f4a68d49a5f42fb6896cd4c2a32 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 12 Dec 2024 13:58:29 -0800 Subject: [PATCH 14/59] update exporter --- .../experimental/otel_tracing/core/exporter.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index 558767cc9..7b266d744 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Sequence from opentelemetry.sdk.trace import ReadableSpan @@ -28,16 +29,23 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: event_id=str(context.span_id), record=span.attributes, record_attributes=span.attributes, - record_type="span", + record_type="SPAN", resource_attributes=span.resource.attributes, - start_timestamp=span.start_time, - timestamp=span.end_time, + start_timestamp=datetime.fromtimestamp( + span.start_time / pow(10, 9) + ) + if span.start_time + else datetime.now(), + timestamp=datetime.fromtimestamp(span.end_time / pow(10, 9)) + if span.end_time + else datetime.now(), trace={ "trace_id": str(context.trace_id), "parent_id": str(context.span_id), }, ) self.connector.add_event(event) + print(f"adding event {str(context.span_id)}") print(event) return SpanExportResult.SUCCESS From d0b43987c2a8e9857efb229fd637df10ac49de4f Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 12 Dec 2024 14:19:39 -0800 Subject: [PATCH 15/59] update --- .../otel_tracing/core/exporter.py | 64 +++++++++++-------- .../experimental/otel_tracing/core/init.py | 23 ++++++- 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index 7b266d744..e94f3cb38 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -1,4 +1,5 @@ from datetime import datetime +import logging from typing import Sequence from opentelemetry.sdk.trace import ReadableSpan @@ -7,6 +8,8 @@ from trulens.core.database import connector as core_connector from trulens.core.schema import event as event_schema +logger = logging.getLogger(__name__) + class TruLensDBSpanExporter(SpanExporter): """ @@ -19,34 +22,41 @@ def __init__(self, connector: core_connector.DBConnector): self.connector = connector def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - for span in spans: - context = span.get_span_context() - - if context is None: - continue - - event = event_schema.Event( - event_id=str(context.span_id), - record=span.attributes, - record_attributes=span.attributes, - record_type="SPAN", - resource_attributes=span.resource.attributes, - start_timestamp=datetime.fromtimestamp( - span.start_time / pow(10, 9) + try: + for span in spans: + context = span.get_span_context() + + if context is None: + logger.error( + "Error exporting spans to the database: Span context is None" + ) + return SpanExportResult.FAILURE + + event = event_schema.Event( + event_id=str(context.span_id), + record=span.attributes, + record_attributes=span.attributes, + record_type="SPAN", + resource_attributes=span.resource.attributes, + start_timestamp=datetime.fromtimestamp( + span.start_time / pow(10, 9) + ) + if span.start_time + else datetime.now(), + timestamp=datetime.fromtimestamp(span.end_time / pow(10, 9)) + if span.end_time + else datetime.now(), + trace={ + "trace_id": str(context.trace_id), + "parent_id": str(context.span_id), + }, ) - if span.start_time - else datetime.now(), - timestamp=datetime.fromtimestamp(span.end_time / pow(10, 9)) - if span.end_time - else datetime.now(), - trace={ - "trace_id": str(context.trace_id), - "parent_id": str(context.span_id), - }, - ) - self.connector.add_event(event) - print(f"adding event {str(context.span_id)}") - print(event) + self.connector.add_event(event) + + except Exception as e: + logger.error("Error exporting spans to the database: %s", e) + return SpanExportResult.FAILURE + return SpanExportResult.SUCCESS """Immediately export all spans""" diff --git a/src/core/trulens/experimental/otel_tracing/core/init.py b/src/core/trulens/experimental/otel_tracing/core/init.py index 9f9a719ac..769812433 100644 --- a/src/core/trulens/experimental/otel_tracing/core/init.py +++ b/src/core/trulens/experimental/otel_tracing/core/init.py @@ -1,3 +1,5 @@ +import logging + from opentelemetry import trace from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider @@ -12,6 +14,9 @@ TRULENS_SERVICE_NAME = "trulens" +logger = logging.getLogger(__name__) + + def init(session: TruSession, debug: bool = False): """Initialize the OpenTelemetry SDK with TruLens configuration.""" resource = Resource.create({"service.name": TRULENS_SERVICE_NAME}) @@ -19,7 +24,7 @@ def init(session: TruSession, debug: bool = False): trace.set_tracer_provider(provider) if debug: - print( + logging.debug( "Initializing OpenTelemetry with TruLens configuration for console debugging" ) # Add a console exporter for debugging purposes @@ -28,9 +33,21 @@ def init(session: TruSession, debug: bool = False): provider.add_span_processor(console_processor) if session.connector: - print("Exporting traces to the TruLens database") + logging.debug("Exporting traces to the TruLens database") - # TODO: Validate that the table exists. + # Check the database revision + try: + db_revision = session.connector.db.get_db_revision() + if db_revision is None: + raise ValueError( + "Database revision is not set. Please run the migrations." + ) + if int(db_revision) < 10: + raise ValueError( + "Database revision is too low. Please run the migrations." + ) + except Exception: + raise ValueError("Error checking the database revision.") # Add the TruLens database exporter db_exporter = TruLensDBSpanExporter(session.connector) From 99dcb44b963589d5562c9ef3c201b9aa7071c90e Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 12 Dec 2024 14:22:35 -0800 Subject: [PATCH 16/59] nits --- src/core/trulens/experimental/otel_tracing/core/init.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/init.py b/src/core/trulens/experimental/otel_tracing/core/init.py index 769812433..aa6f0f67e 100644 --- a/src/core/trulens/experimental/otel_tracing/core/init.py +++ b/src/core/trulens/experimental/otel_tracing/core/init.py @@ -3,7 +3,6 @@ from opentelemetry import trace from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.trace.export import ConsoleSpanExporter from opentelemetry.sdk.trace.export import SimpleSpanProcessor from trulens.core.session import TruSession @@ -51,5 +50,5 @@ def init(session: TruSession, debug: bool = False): # Add the TruLens database exporter db_exporter = TruLensDBSpanExporter(session.connector) - db_processor = BatchSpanProcessor(db_exporter) + db_processor = SimpleSpanProcessor(db_exporter) provider.add_span_processor(db_processor) From 834d6e7e2519e15cbaea14123bb731b9b55483e9 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Mon, 16 Dec 2024 16:27:15 -0500 Subject: [PATCH 17/59] fix parent --- src/core/trulens/experimental/otel_tracing/core/exporter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index e94f3cb38..4549906e4 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -25,6 +25,7 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: try: for span in spans: context = span.get_span_context() + parent = span.parent if context is None: logger.error( @@ -48,7 +49,7 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: else datetime.now(), trace={ "trace_id": str(context.trace_id), - "parent_id": str(context.span_id), + "parent_id": str(parent.span_id if parent else ""), }, ) self.connector.add_event(event) From eb61153d1b2bfbf431c661caa54a759b6081504c Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:09:52 -0500 Subject: [PATCH 18/59] save --- src/core/trulens/core/database/base.py | 11 ++++++++++- src/core/trulens/core/database/connector/base.py | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/core/trulens/core/database/base.py b/src/core/trulens/core/database/base.py index 36b76b006..89f1e82ac 100644 --- a/src/core/trulens/core/database/base.py +++ b/src/core/trulens/core/database/base.py @@ -91,6 +91,15 @@ def check_db_revision(self): """ raise NotImplementedError() + @abc.abstractmethod + def get_db_dialect(self) -> Optional[str]: + """Get the dialect of the database. + + Returns: + The dialect of the database. + """ + raise NotImplementedError() + @abc.abstractmethod def get_db_revision(self) -> Optional[str]: """Get the current revision of the database. @@ -435,7 +444,7 @@ def get_datasets(self) -> pd.DataFrame: raise NotImplementedError() @abc.abstractmethod - def insert_event(self, event: event_schema.Event) -> types_schema.EventID: + def insert_event(self, event: event_schema.Event) -> event_schema.EventID: """Insert an event into the database. Args: diff --git a/src/core/trulens/core/database/connector/base.py b/src/core/trulens/core/database/connector/base.py index 3a181c045..9f32cb283 100644 --- a/src/core/trulens/core/database/connector/base.py +++ b/src/core/trulens/core/database/connector/base.py @@ -261,6 +261,7 @@ def add_feedbacks( ], ) -> List[types_schema.FeedbackResultID]: """Add multiple feedback results to the database and return their unique ids. + # TODO: This is slow and should be batched or otherwise optimized in the future. Args: feedback_results: An iterable with each iteration being a [FeedbackResult][trulens.core.schema.feedback.FeedbackResult] or @@ -422,6 +423,7 @@ def add_event(self, event: event_schema.Event): def add_events(self, events: List[event_schema.Event]): """ Add multiple events to the database. + # TODO: This is slow and should be batched or otherwise optimized in the future. Args: events: A list of events to add to the database. From 7fd7de277f017efb86a2817073e7dd67ee2f996a Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:16:51 -0500 Subject: [PATCH 19/59] save --- .../otel_tracing/core/exporter.py | 67 +++++++++---------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index 4549906e4..a9e9f6370 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -1,6 +1,6 @@ from datetime import datetime import logging -from typing import Sequence +from typing import Optional, Sequence from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import SpanExporter @@ -11,6 +11,13 @@ logger = logging.getLogger(__name__) +def to_timestamp(timestamp: Optional[int]) -> datetime: + if timestamp: + return datetime.fromtimestamp(timestamp * 1e-9) + + return datetime.now() + + class TruLensDBSpanExporter(SpanExporter): """ Implementation of :class:`SpanExporter` that flushes the spans to the database in the TruLens session. @@ -21,46 +28,34 @@ class TruLensDBSpanExporter(SpanExporter): def __init__(self, connector: core_connector.DBConnector): self.connector = connector + def _construct_event(self, span: ReadableSpan) -> event_schema.Event: + context = span.get_span_context() + parent = span.parent + + if context is None: + raise ValueError("Span context is None") + + return event_schema.Event( + record=span.attributes, + record_attributes={}, + record_type=event_schema.EventRecordType.SPAN, + resource_attributes=span.resource.attributes, + start_timestamp=to_timestamp(span.start_time), + timestamp=to_timestamp(span.end_time), + trace={ + "span_id": str(context.span_id), + "trace_id": str(context.trace_id), + "parent_id": str(parent.span_id if parent else ""), + }, + ) + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: try: - for span in spans: - context = span.get_span_context() - parent = span.parent - - if context is None: - logger.error( - "Error exporting spans to the database: Span context is None" - ) - return SpanExportResult.FAILURE - - event = event_schema.Event( - event_id=str(context.span_id), - record=span.attributes, - record_attributes=span.attributes, - record_type="SPAN", - resource_attributes=span.resource.attributes, - start_timestamp=datetime.fromtimestamp( - span.start_time / pow(10, 9) - ) - if span.start_time - else datetime.now(), - timestamp=datetime.fromtimestamp(span.end_time / pow(10, 9)) - if span.end_time - else datetime.now(), - trace={ - "trace_id": str(context.trace_id), - "parent_id": str(parent.span_id if parent else ""), - }, - ) - self.connector.add_event(event) + events = list(map(self._construct_event, spans)) + self.connector.add_events(events) except Exception as e: logger.error("Error exporting spans to the database: %s", e) return SpanExportResult.FAILURE return SpanExportResult.SUCCESS - - """Immediately export all spans""" - - def force_flush(self, timeout_millis: int = 30000) -> bool: - return True From aad3ffa67b568be2615723213b6d84a9d322544e Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:20:34 -0500 Subject: [PATCH 20/59] update typing --- src/core/trulens/core/database/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/trulens/core/database/base.py b/src/core/trulens/core/database/base.py index 89f1e82ac..26d551fa9 100644 --- a/src/core/trulens/core/database/base.py +++ b/src/core/trulens/core/database/base.py @@ -444,7 +444,7 @@ def get_datasets(self) -> pd.DataFrame: raise NotImplementedError() @abc.abstractmethod - def insert_event(self, event: event_schema.Event) -> event_schema.EventID: + def insert_event(self, event: event_schema.Event) -> types_schema.EventID: """Insert an event into the database. Args: From 9e27b0f63632eb16a5334ea35b4aa9d9a7b73257 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:31:30 -0500 Subject: [PATCH 21/59] update --- src/core/trulens/core/database/sqlalchemy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/trulens/core/database/sqlalchemy.py b/src/core/trulens/core/database/sqlalchemy.py index 0aca9c3c0..05831cf30 100644 --- a/src/core/trulens/core/database/sqlalchemy.py +++ b/src/core/trulens/core/database/sqlalchemy.py @@ -978,7 +978,7 @@ def insert_event(self, event: Event) -> types_schema.EventID: """See [DB.insert_event][trulens.core.database.base.DB.insert_event].""" with self.session.begin() as session: _event = self.orm.Event.parse(event, redact_keys=self.redact_keys) - session.merge(_event) + session.add(_event) logger.info( f"{text_utils.UNICODE_CHECK} added event {_event.event_id}" ) From 8004a55a5a77e3d448246a1846e122fac214ec16 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 19 Dec 2024 15:22:58 -0500 Subject: [PATCH 22/59] update --- src/core/trulens/experimental/otel_tracing/core/exporter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index a9e9f6370..358d45bd9 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -36,6 +36,7 @@ def _construct_event(self, span: ReadableSpan) -> event_schema.Event: raise ValueError("Span context is None") return event_schema.Event( + event_id=str(context.span_id), record=span.attributes, record_attributes={}, record_type=event_schema.EventRecordType.SPAN, From f35f2b052e44e7c490981ba169dc2f171983fc46 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 18:18:02 -0500 Subject: [PATCH 23/59] update exporter to have the attributes in attributes --- .../experimental/otel_tracing/core/exporter.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index 358d45bd9..9c44b9eeb 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -5,6 +5,7 @@ from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import SpanExporter from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.trace import StatusCode from trulens.core.database import connector as core_connector from trulens.core.schema import event as event_schema @@ -20,7 +21,7 @@ def to_timestamp(timestamp: Optional[int]) -> datetime: class TruLensDBSpanExporter(SpanExporter): """ - Implementation of :class:`SpanExporter` that flushes the spans to the database in the TruLens session. + Implementation of `SpanExporter` that flushes the spans to the database in the TruLens session. """ connector: core_connector.DBConnector @@ -37,8 +38,15 @@ def _construct_event(self, span: ReadableSpan) -> event_schema.Event: return event_schema.Event( event_id=str(context.span_id), - record=span.attributes, - record_attributes={}, + record={ + "name": span.name, + "kind": "SPAN_KIND_TRULENS", + "parent_span_id": str(parent.span_id if parent else ""), + "status": "STATUS_CODE_ERROR" + if span.status.status_code == StatusCode.ERROR + else "STATUS_CODE_UNSET", + }, + record_attributes=span.attributes, record_type=event_schema.EventRecordType.SPAN, resource_attributes=span.resource.attributes, start_timestamp=to_timestamp(span.start_time), From 4ed45464058a368c34f981a1fb0bff70393a2f5a Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 11 Dec 2024 16:07:10 -0800 Subject: [PATCH 24/59] save --- examples/experimental/otel_exporter.ipynb | 91 ++++++++++++++++++- .../otel_tracing/core/instrument.py | 62 +++++++++++++ 2 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 src/core/trulens/experimental/otel_tracing/core/instrument.py diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index fb00a0180..940f3cda6 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -124,6 +124,93 @@ "pygments_lexer": "ipython3" } }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Add base dir to path to be able to access test folder.\n", + "base_dir = Path().cwd().parent.parent.resolve()\n", + "if str(base_dir) not in sys.path:\n", + " print(f\"Adding {base_dir} to sys.path\")\n", + " sys.path.append(str(base_dir))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.experimental.otel_tracing.core.instrument import instrument\n", + "\n", + "from examples.dev.dummy_app.dummy import Dummy\n", + "\n", + "\n", + "class TestApp(Dummy):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " @instrument\n", + " def respond_to_query(self, query: str) -> str:\n", + " return f\"answer: {self.nested(query)}\"\n", + "\n", + " @instrument\n", + " def nested(self, query: str) -> str:\n", + " return f\"nested: {query}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.core.session import TruSession\n", + "from trulens.experimental.otel_tracing.core.init import init\n", + "\n", + "session = TruSession()\n", + "init(session, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.apps.custom import TruCustomApp\n", + "\n", + "test_app = TestApp()\n", + "custom_app = TruCustomApp(test_app)\n", + "\n", + "with custom_app as recording:\n", + " test_app.respond_to_query(\"test\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "trulens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py new file mode 100644 index 000000000..b006bd564 --- /dev/null +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -0,0 +1,62 @@ +from typing import Callable + +from opentelemetry import trace +from trulens.apps.custom import TruCustomApp +from trulens.apps.custom import instrument as custom_instrument +from trulens.core import instruments as core_instruments +from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME + + +class instrument2(core_instruments.instrument): + """ + Decorator for marking methods to be instrumented in custom classes that are + wrapped by TruCustomApp, with OpenTelemetry tracing. + """ + + @classmethod + def method(cls, inst_cls: type, name: str) -> None: + core_instruments.instrument.method(inst_cls, name) + + # Also make note of it for verification that it was found by the walk + # after init. + TruCustomApp.functions_to_instrument.add(getattr(inst_cls, name)) + + # `_self` is used to avoid conflicts where `self` may be passed from the caller method + def __call__(_self, *args, **kwargs): + print("in call") + with ( + trace.get_tracer_provider() + .get_tracer(TRULENS_SERVICE_NAME) + .start_as_current_span( + name=_self.func.__name__, + ) + ) as span: + span.set_attribute("function", _self.func.__name__) + span.set_attribute("args", args) + span.set_attribute("kwargs", **kwargs) + ret = super.__call__(_self, *args, **kwargs) + span.set_attribute("return", ret) + return ret + + +def instrument(func: Callable): + """ + Decorator for marking functions to be instrumented in custom classes that are + wrapped by TruCustomApp, with OpenTelemetry tracing. + """ + + def wrapper(*args, **kwargs): + with ( + trace.get_tracer_provider() + .get_tracer(TRULENS_SERVICE_NAME) + .start_as_current_span( + name=func.__name__, + ) + ) as span: + span.set_attribute("function", func.__name__) + span.set_attribute("args", args) + ret = custom_instrument(func)(*args, **kwargs) + span.set_attribute("return", ret) + return ret + + return wrapper From c1516c7f1eea931a5d9e543789c5ef472ac660e4 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 11 Dec 2024 16:39:46 -0800 Subject: [PATCH 25/59] save --- examples/experimental/otel_exporter.ipynb | 7 +- .../otel_tracing/core/instrument.py | 80 +++++++++++-------- .../otel_tracing/core/semantic.py | 1 + 3 files changed, 47 insertions(+), 41 deletions(-) create mode 100644 src/core/trulens/experimental/otel_tracing/core/semantic.py diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 940f3cda6..77cded69c 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -148,13 +148,8 @@ "source": [ "from trulens.experimental.otel_tracing.core.instrument import instrument\n", "\n", - "from examples.dev.dummy_app.dummy import Dummy\n", - "\n", - "\n", - "class TestApp(Dummy):\n", - " def __init__(self):\n", - " super().__init__()\n", "\n", + "class TestApp:\n", " @instrument\n", " def respond_to_query(self, query: str) -> str:\n", " return f\"answer: {self.nested(query)}\"\n", diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index b006bd564..35959f8ea 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -1,50 +1,42 @@ -from typing import Callable +from functools import wraps +import logging +from typing import Any, Callable, Union from opentelemetry import trace -from trulens.apps.custom import TruCustomApp from trulens.apps.custom import instrument as custom_instrument -from trulens.core import instruments as core_instruments +from trulens.core.utils import json as json_utils from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME +from trulens.experimental.otel_tracing.core.semantic import ( + TRULENS_SELECTOR_NAME, +) +logger = logging.getLogger(__name__) -class instrument2(core_instruments.instrument): - """ - Decorator for marking methods to be instrumented in custom classes that are - wrapped by TruCustomApp, with OpenTelemetry tracing. - """ - - @classmethod - def method(cls, inst_cls: type, name: str) -> None: - core_instruments.instrument.method(inst_cls, name) - - # Also make note of it for verification that it was found by the walk - # after init. - TruCustomApp.functions_to_instrument.add(getattr(inst_cls, name)) - - # `_self` is used to avoid conflicts where `self` may be passed from the caller method - def __call__(_self, *args, **kwargs): - print("in call") - with ( - trace.get_tracer_provider() - .get_tracer(TRULENS_SERVICE_NAME) - .start_as_current_span( - name=_self.func.__name__, - ) - ) as span: - span.set_attribute("function", _self.func.__name__) - span.set_attribute("args", args) - span.set_attribute("kwargs", **kwargs) - ret = super.__call__(_self, *args, **kwargs) - span.set_attribute("return", ret) - return ret +type Attributes = Union[ + dict[str, Any], Callable[[Any, Any, Any], dict[str, Any]] +] -def instrument(func: Callable): +def instrument(func: Callable, attributes: Attributes = {}): """ Decorator for marking functions to be instrumented in custom classes that are wrapped by TruCustomApp, with OpenTelemetry tracing. """ + def _validate_selector_name(final_attributes: dict[str, Any]): + if TRULENS_SELECTOR_NAME in final_attributes: + selector_name = final_attributes[TRULENS_SELECTOR_NAME] + if not isinstance(selector_name, str): + raise ValueError( + f"Selector name must be a string, not {type(selector_name)}" + ) + + def _validate_attributes(final_attributes: dict[str, Any]): + _validate_selector_name(final_attributes) + # TODO: validate OTEL attributes. + # TODO: validate span type attributes. + + @wraps(func) def wrapper(*args, **kwargs): with ( trace.get_tracer_provider() @@ -53,10 +45,28 @@ def wrapper(*args, **kwargs): name=func.__name__, ) ) as span: + _instrumented_object, *rest_args = args + + # ? Do we have to validate the args/kwargs and make sure they are serializable? span.set_attribute("function", func.__name__) - span.set_attribute("args", args) + span.set_attribute("args", json_utils.json_str_of_obj(rest_args)) ret = custom_instrument(func)(*args, **kwargs) span.set_attribute("return", ret) + + attributes_to_add = {} + + # Set the user provider attributes. + if attributes: + if callable(attributes): + attributes_to_add = attributes(ret, *args, **kwargs) + else: + attributes_to_add = attributes + + _validate_attributes(attributes_to_add) + + for key, value in attributes_to_add.items(): + span.set_attribute(key, value) + return ret return wrapper diff --git a/src/core/trulens/experimental/otel_tracing/core/semantic.py b/src/core/trulens/experimental/otel_tracing/core/semantic.py new file mode 100644 index 000000000..d66bd00b8 --- /dev/null +++ b/src/core/trulens/experimental/otel_tracing/core/semantic.py @@ -0,0 +1 @@ +TRULENS_SELECTOR_NAME = "trulens.selector_name" From 85f2e6239ae621f547b71d904e11a6e88230d60b Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 11 Dec 2024 16:54:42 -0800 Subject: [PATCH 26/59] update --- examples/experimental/otel_exporter.ipynb | 15 +++- .../otel_tracing/core/instrument.py | 72 +++++++++++-------- 2 files changed, 53 insertions(+), 34 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 77cded69c..070f5b552 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -150,13 +150,22 @@ "\n", "\n", "class TestApp:\n", - " @instrument\n", + " @instrument()\n", " def respond_to_query(self, query: str) -> str:\n", " return f\"answer: {self.nested(query)}\"\n", "\n", - " @instrument\n", + " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", " def nested(self, query: str) -> str:\n", - " return f\"nested: {query}\"" + " return f\"nested: {self.nested2(query)}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, *args, **kwargs: {\n", + " \"nested2_ret\": ret,\n", + " \"nested2_args[0]\": args[0],\n", + " }\n", + " )\n", + " def nested2(self, query: str) -> str:\n", + " return f\"nested2: {query}\"" ] }, { diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 35959f8ea..ed4e50bed 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -1,6 +1,6 @@ from functools import wraps import logging -from typing import Any, Callable, Union +from typing import Any, Callable, Optional, Union from opentelemetry import trace from trulens.apps.custom import instrument as custom_instrument @@ -12,12 +12,12 @@ logger = logging.getLogger(__name__) -type Attributes = Union[ - dict[str, Any], Callable[[Any, Any, Any], dict[str, Any]] +type Attributes = Optional[ + Union[dict[str, Any], Callable[[Any, Any, Any], dict[str, Any]]] ] -def instrument(func: Callable, attributes: Attributes = {}): +def instrument(attributes: Attributes = {}): """ Decorator for marking functions to be instrumented in custom classes that are wrapped by TruCustomApp, with OpenTelemetry tracing. @@ -36,37 +36,47 @@ def _validate_attributes(final_attributes: dict[str, Any]): # TODO: validate OTEL attributes. # TODO: validate span type attributes. - @wraps(func) - def wrapper(*args, **kwargs): - with ( - trace.get_tracer_provider() - .get_tracer(TRULENS_SERVICE_NAME) - .start_as_current_span( - name=func.__name__, - ) - ) as span: - _instrumented_object, *rest_args = args + def inner_decorator(func: Callable): + @wraps(func) + def wrapper(*args, **kwargs): + with ( + trace.get_tracer_provider() + .get_tracer(TRULENS_SERVICE_NAME) + .start_as_current_span( + name=func.__name__, + ) + ): + span = trace.get_current_span() + _instrumented_object, *rest_args = args + + # ? Do we have to validate the args/kwargs and make sure they are serializable? + span.set_attribute("function", func.__name__) + span.set_attribute( + "args", json_utils.json_str_of_obj(rest_args) + ) + ret = custom_instrument(func)(*args, **kwargs) + span.set_attribute("return", ret) + + attributes_to_add = {} - # ? Do we have to validate the args/kwargs and make sure they are serializable? - span.set_attribute("function", func.__name__) - span.set_attribute("args", json_utils.json_str_of_obj(rest_args)) - ret = custom_instrument(func)(*args, **kwargs) - span.set_attribute("return", ret) + # Set the user provider attributes. + if attributes: + if callable(attributes): + attributes_to_add = attributes( + ret, *rest_args, **kwargs + ) + else: + attributes_to_add = attributes - attributes_to_add = {} + logger.info(f"Attributes to add: {attributes_to_add}") - # Set the user provider attributes. - if attributes: - if callable(attributes): - attributes_to_add = attributes(ret, *args, **kwargs) - else: - attributes_to_add = attributes + _validate_attributes(attributes_to_add) - _validate_attributes(attributes_to_add) + for key, value in attributes_to_add.items(): + span.set_attribute(key, value) - for key, value in attributes_to_add.items(): - span.set_attribute(key, value) + return ret - return ret + return wrapper - return wrapper + return inner_decorator From 47672f2cc5aff170f4e398c46d6779b7b2923b91 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 12 Dec 2024 14:47:54 -0800 Subject: [PATCH 27/59] add try-catch --- .../otel_tracing/core/instrument.py | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index ed4e50bed..a963e1751 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -4,7 +4,6 @@ from opentelemetry import trace from trulens.apps.custom import instrument as custom_instrument -from trulens.core.utils import json as json_utils from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME from trulens.experimental.otel_tracing.core.semantic import ( TRULENS_SELECTOR_NAME, @@ -45,36 +44,42 @@ def wrapper(*args, **kwargs): .start_as_current_span( name=func.__name__, ) - ): + ) as parent_span: span = trace.get_current_span() _instrumented_object, *rest_args = args - # ? Do we have to validate the args/kwargs and make sure they are serializable? - span.set_attribute("function", func.__name__) + span.set_attribute("name", func.__name__) + span.set_attribute("kind", "SPAN_KIND_TRULENS") span.set_attribute( - "args", json_utils.json_str_of_obj(rest_args) + "parent_span_id", parent_span.get_span_context().span_id ) - ret = custom_instrument(func)(*args, **kwargs) - span.set_attribute("return", ret) - attributes_to_add = {} + try: + ret = custom_instrument(func)(*args, **kwargs) - # Set the user provider attributes. - if attributes: - if callable(attributes): - attributes_to_add = attributes( - ret, *rest_args, **kwargs - ) - else: - attributes_to_add = attributes + attributes_to_add = {} - logger.info(f"Attributes to add: {attributes_to_add}") + # Set the user provider attributes. + if attributes: + if callable(attributes): + attributes_to_add = attributes( + ret, *rest_args, **kwargs + ) + else: + attributes_to_add = attributes - _validate_attributes(attributes_to_add) + logger.info(f"Attributes to add: {attributes_to_add}") - for key, value in attributes_to_add.items(): - span.set_attribute(key, value) + _validate_attributes(attributes_to_add) + for key, value in attributes_to_add.items(): + span.set_attribute(key, value) + + except Exception: + span.set_attribute("status", "STATUS_CODE_ERROR") + return None + + span.set_attribute("status", "STATUS_CODE_UNSET") return ret return wrapper From aafae55f8afc7bb526bf008e4c3c68f1f9f36b3c Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 15:44:17 -0500 Subject: [PATCH 28/59] updatE --- examples/experimental/otel_exporter.ipynb | 349 ++++++++++-------- src/core/trulens/core/database/sqlalchemy.py | 50 +++ .../otel_tracing/core/exporter.py | 1 + .../otel_tracing/core/instrument.py | 10 +- 4 files changed, 259 insertions(+), 151 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 070f5b552..34f15ecce 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,9 +12,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Adding /Users/gtokernliang/projects/trulens to sys.path\n" + ] + } + ], "source": [ "from pathlib import Path\n", "import sys\n", @@ -28,81 +36,220 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from typing import Callable\n", - "\n", - "from opentelemetry import trace\n", - "from trulens.apps.custom import instrument\n", - "from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME\n", - "\n", - "\n", - "def decorator(func: Callable):\n", - " tracer = trace.get_tracer(TRULENS_SERVICE_NAME)\n", - "\n", - " def wrapper(*args, **kwargs):\n", - " print(\"start wrap\")\n", - "\n", - " with tracer.start_as_current_span(\"custom\"):\n", - " result = func(*args, **kwargs)\n", - " span = trace.get_current_span()\n", - " print(\"---span---\")\n", - " print(span.get_span_context())\n", - " span.set_attribute(\"result\", result)\n", - " span.set_status(trace.Status(trace.StatusCode.OK))\n", - " return result\n", - "\n", - " return wrapper" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "from examples.dev.dummy_app.dummy import Dummy\n", - "\n", + "from trulens.experimental.otel_tracing.core.instrument import instrument\n", "\n", - "class TestApp(Dummy):\n", - " def __init__(self):\n", - " super().__init__()\n", "\n", - " @decorator\n", - " @instrument\n", + "class TestApp:\n", + " @instrument()\n", " def respond_to_query(self, query: str) -> str:\n", " return f\"answer: {self.nested(query)}\"\n", "\n", - " @decorator\n", - " @instrument\n", + " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", " def nested(self, query: str) -> str:\n", - " return f\"nested: {query}\"" + " return f\"nested: {self.nested2(query)}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, *args, **kwargs: {\n", + " \"nested2_ret\": ret,\n", + " \"nested2_args[0]\": args[0],\n", + " }\n", + " )\n", + " def nested2(self, query: str) -> str:\n", + " return f\"nested2: {query}\"" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦑 Initialized with db url snowflake://gtokernliang:***@fab02971/GTOK/PUBLIC?role=ENGINEER&warehouse=DKUROKAWA .\n", + "🔒 Secret keys will not be included in the database.\n", + "Set TruLens workspace version tag: [('Statement executed successfully.',)]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Updating app_name and app_version in apps table: 0it [00:00, ?it/s]\n", + "Updating app_id in records table: 0it [00:00, ?it/s]\n", + "Updating app_json in apps table: 0it [00:00, ?it/s]\n" + ] + } + ], "source": [ + "import os\n", + "\n", + "import dotenv\n", + "from trulens.connectors.snowflake import SnowflakeConnector\n", "from trulens.core.session import TruSession\n", "from trulens.experimental.otel_tracing.core.init import init\n", "\n", - "session = TruSession()\n", - "init(session)" + "dotenv.load_dotenv()\n", + "\n", + "connection_params = {\n", + " \"account\": os.environ[\"SNOWFLAKE_ACCOUNT\"],\n", + " \"user\": os.environ[\"SNOWFLAKE_USER\"],\n", + " \"password\": os.environ[\"SNOWFLAKE_USER_PASSWORD\"],\n", + " \"database\": os.environ[\"SNOWFLAKE_DATABASE\"],\n", + " \"schema\": os.environ[\"SNOWFLAKE_SCHEMA\"],\n", + " \"warehouse\": os.environ[\"SNOWFLAKE_WAREHOUSE\"],\n", + " \"role\": os.environ[\"SNOWFLAKE_ROLE\"],\n", + "}\n", + "\n", + "connector = SnowflakeConnector(\n", + " **connection_params, database_redact_keys=True, database_args=None\n", + ")\n", + "session = TruSession(connector=connector)\n", + "session.reset_database()\n", + "init(session, debug=True)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "skipping base because of class\n", + "skipping base because of class\n", + "decorating \n", + "decorating \n", + "decorating \n", + "{\n", + " \"name\": \"nested2\",\n", + " \"context\": {\n", + " \"trace_id\": \"0x4ed62c9e990bf71443c0bcedf0f719cc\",\n", + " \"span_id\": \"0x324e378d20841912\",\n", + " \"trace_state\": \"[]\"\n", + " },\n", + " \"kind\": \"SpanKind.INTERNAL\",\n", + " \"parent_id\": \"0xae573ae47e6ebc11\",\n", + " \"start_time\": \"2024-12-18T22:32:52.897168Z\",\n", + " \"end_time\": \"2024-12-18T22:32:52.897215Z\",\n", + " \"status\": {\n", + " \"status_code\": \"UNSET\"\n", + " },\n", + " \"attributes\": {\n", + " \"name\": \"nested2\",\n", + " \"kind\": \"SPAN_KIND_TRULENS\",\n", + " \"parent_span_id\": 3624895829355272466,\n", + " \"nested2_ret\": \"nested2: test\",\n", + " \"nested2_args[0]\": \"test\",\n", + " \"status\": \"STATUS_CODE_UNSET\"\n", + " },\n", + " \"events\": [],\n", + " \"links\": [],\n", + " \"resource\": {\n", + " \"attributes\": {\n", + " \"telemetry.sdk.language\": \"python\",\n", + " \"telemetry.sdk.name\": \"opentelemetry\",\n", + " \"telemetry.sdk.version\": \"1.25.0\",\n", + " \"service.name\": \"trulens\"\n", + " },\n", + " \"schema_url\": \"\"\n", + " }\n", + "}\n", + "Error exporting spans to the database: %s 1 validation error for Event\n", + "event_id\n", + " Field required [type=missing, input_value={'record': mappingproxy({...'12562574438621428753'}}, input_type=dict]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/missing\n", + "{\n", + " \"name\": \"nested\",\n", + " \"context\": {\n", + " \"trace_id\": \"0x4ed62c9e990bf71443c0bcedf0f719cc\",\n", + " \"span_id\": \"0xae573ae47e6ebc11\",\n", + " \"trace_state\": \"[]\"\n", + " },\n", + " \"kind\": \"SpanKind.INTERNAL\",\n", + " \"parent_id\": \"0x7cfd5e50f405e5be\",\n", + " \"start_time\": \"2024-12-18T22:32:52.897083Z\",\n", + " \"end_time\": \"2024-12-18T22:32:52.897914Z\",\n", + " \"status\": {\n", + " \"status_code\": \"UNSET\"\n", + " },\n", + " \"attributes\": {\n", + " \"name\": \"nested\",\n", + " \"kind\": \"SPAN_KIND_TRULENS\",\n", + " \"parent_span_id\": 12562574438621428753,\n", + " \"nested_attr1\": \"value1\",\n", + " \"status\": \"STATUS_CODE_UNSET\"\n", + " },\n", + " \"events\": [],\n", + " \"links\": [],\n", + " \"resource\": {\n", + " \"attributes\": {\n", + " \"telemetry.sdk.language\": \"python\",\n", + " \"telemetry.sdk.name\": \"opentelemetry\",\n", + " \"telemetry.sdk.version\": \"1.25.0\",\n", + " \"service.name\": \"trulens\"\n", + " },\n", + " \"schema_url\": \"\"\n", + " }\n", + "}\n", + "Error exporting spans to the database: %s 1 validation error for Event\n", + "event_id\n", + " Field required [type=missing, input_value={'record': mappingproxy({... '9006458531595281854'}}, input_type=dict]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/missing\n", + "{\n", + " \"name\": \"respond_to_query\",\n", + " \"context\": {\n", + " \"trace_id\": \"0x4ed62c9e990bf71443c0bcedf0f719cc\",\n", + " \"span_id\": \"0x7cfd5e50f405e5be\",\n", + " \"trace_state\": \"[]\"\n", + " },\n", + " \"kind\": \"SpanKind.INTERNAL\",\n", + " \"parent_id\": null,\n", + " \"start_time\": \"2024-12-18T22:32:52.896826Z\",\n", + " \"end_time\": \"2024-12-18T22:32:52.898300Z\",\n", + " \"status\": {\n", + " \"status_code\": \"UNSET\"\n", + " },\n", + " \"attributes\": {\n", + " \"name\": \"respond_to_query\",\n", + " \"kind\": \"SPAN_KIND_TRULENS\",\n", + " \"parent_span_id\": 9006458531595281854,\n", + " \"status\": \"STATUS_CODE_UNSET\"\n", + " },\n", + " \"events\": [],\n", + " \"links\": [],\n", + " \"resource\": {\n", + " \"attributes\": {\n", + " \"telemetry.sdk.language\": \"python\",\n", + " \"telemetry.sdk.name\": \"opentelemetry\",\n", + " \"telemetry.sdk.version\": \"1.25.0\",\n", + " \"service.name\": \"trulens\"\n", + " },\n", + " \"schema_url\": \"\"\n", + " }\n", + "}\n", + "Error exporting spans to the database: %s 1 validation error for Event\n", + "event_id\n", + " Field required [type=missing, input_value={'record': mappingproxy({...4540', 'parent_id': ''}}, input_type=dict]\n", + " For further information visit https://errors.pydantic.dev/2.10/v/missing\n" + ] + } + ], "source": [ + "from trulens.apps.custom import TruCustomApp\n", + "\n", "test_app = TestApp()\n", + "custom_app = TruCustomApp(test_app)\n", "\n", - "test_app.respond_to_query(\"test\")" + "with custom_app as recording:\n", + " test_app.respond_to_query(\"test\")" ] } ], @@ -121,100 +268,10 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.11" } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import sys\n", - "\n", - "# Add base dir to path to be able to access test folder.\n", - "base_dir = Path().cwd().parent.parent.resolve()\n", - "if str(base_dir) not in sys.path:\n", - " print(f\"Adding {base_dir} to sys.path\")\n", - " sys.path.append(str(base_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.experimental.otel_tracing.core.instrument import instrument\n", - "\n", - "\n", - "class TestApp:\n", - " @instrument()\n", - " def respond_to_query(self, query: str) -> str:\n", - " return f\"answer: {self.nested(query)}\"\n", - "\n", - " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", - " def nested(self, query: str) -> str:\n", - " return f\"nested: {self.nested2(query)}\"\n", - "\n", - " @instrument(\n", - " attributes=lambda ret, *args, **kwargs: {\n", - " \"nested2_ret\": ret,\n", - " \"nested2_args[0]\": args[0],\n", - " }\n", - " )\n", - " def nested2(self, query: str) -> str:\n", - " return f\"nested2: {query}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.core.session import TruSession\n", - "from trulens.experimental.otel_tracing.core.init import init\n", - "\n", - "session = TruSession()\n", - "init(session, debug=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.apps.custom import TruCustomApp\n", - "\n", - "test_app = TestApp()\n", - "custom_app = TruCustomApp(test_app)\n", - "\n", - "with custom_app as recording:\n", - " test_app.respond_to_query(\"test\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "trulens", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/src/core/trulens/core/database/sqlalchemy.py b/src/core/trulens/core/database/sqlalchemy.py index 05831cf30..5544707dd 100644 --- a/src/core/trulens/core/database/sqlalchemy.py +++ b/src/core/trulens/core/database/sqlalchemy.py @@ -25,11 +25,15 @@ import pandas as pd import pydantic from pydantic import Field +from snowflake.sqlalchemy import dialect as SnowflakeDialect import sqlalchemy as sa from sqlalchemy import Table +from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import joinedload from sqlalchemy.orm import sessionmaker from sqlalchemy.sql import text as sql_text +from sqlalchemy.sql.compiler import SQLCompiler +from sqlalchemy.sql.expression import Insert from trulens.core.database import base as core_db from trulens.core.database import exceptions as db_exceptions from trulens.core.database import migrations as db_migrations @@ -40,6 +44,7 @@ from trulens.core.schema import app as app_schema from trulens.core.schema import base as base_schema from trulens.core.schema import dataset as dataset_schema +from trulens.core.schema import event as event_schema from trulens.core.schema import feedback as feedback_schema from trulens.core.schema import groundtruth as groundtruth_schema from trulens.core.schema import record as record_schema @@ -53,6 +58,47 @@ logger = logging.getLogger(__name__) +@compiles(Insert, SnowflakeDialect.name) +def patch_insert(statement: Insert, compiler: SQLCompiler, **kw): + """ + Patches INSERT SQL queries so sqlalchemy ORM will support Snowflake OBJECT. + + See: + * https://github.com/snowflakedb/snowflake-sqlalchemy/issues/299 + * https://github.com/snowflakedb/snowflake-sqlalchemy/issues/411 + + For more information (e.g. read about the parameters), please look at: + * https://docs.sqlalchemy.org/en/20/core/compiler.html + """ + insert_statement = compiler.visit_insert(statement, **kw) + + if statement.table.name.endswith("_events"): + insert_statement = insert_statement.replace( + "VALUES (%(record)s, %(record_attributes)s, %(record_type)s, %(resource_attributes)s, %(start_timestamp)s, %(timestamp)s, %(trace)s)", + """ +SELECT + PARSE_JSON(column1), + PARSE_JSON(column2), + column3, + PARSE_JSON(column4), + column5, + column6, + PARSE_JSON(column7), +from VALUES ( + %(record)s, + %(record_attributes)s, + %(record_type)s, + %(resource_attributes)s, + %(start_timestamp)s, + %(timestamp)s, + %(trace)s +) +""", + ) + + return insert_statement + + class SnowflakeImpl(DefaultImpl): __dialect__ = "snowflake" @@ -251,6 +297,9 @@ def check_db_revision(self): db_utils.check_db_revision(self.engine, self.table_prefix) + def get_db_dialect(self) -> Optional[str]: + return self.engine.dialect.name if self.engine else None + def get_db_revision(self) -> Optional[str]: if self.engine is None: raise ValueError("Database engine not initialized.") @@ -976,6 +1025,7 @@ def get_datasets(self) -> pd.DataFrame: def insert_event(self, event: Event) -> types_schema.EventID: """See [DB.insert_event][trulens.core.database.base.DB.insert_event].""" + with self.session.begin() as session: _event = self.orm.Event.parse(event, redact_keys=self.redact_keys) session.add(_event) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index 9c44b9eeb..a82d3d408 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -65,6 +65,7 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: except Exception as e: logger.error("Error exporting spans to the database: %s", e) + print("Error exporting spans to the database: %s", e) return SpanExportResult.FAILURE return SpanExportResult.SUCCESS diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index a963e1751..acc9d76e0 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -11,12 +11,12 @@ logger = logging.getLogger(__name__) -type Attributes = Optional[ - Union[dict[str, Any], Callable[[Any, Any, Any], dict[str, Any]]] -] - -def instrument(attributes: Attributes = {}): +def instrument( + attributes: Optional[ + Union[dict[str, Any], Callable[[Any, Any, Any], dict[str, Any]]] + ] = {}, +): """ Decorator for marking functions to be instrumented in custom classes that are wrapped by TruCustomApp, with OpenTelemetry tracing. From 3d23e0b1d3d8e428827600038ad80adb30501d55 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 15:51:40 -0500 Subject: [PATCH 29/59] PR feedback --- src/core/trulens/experimental/otel_tracing/core/exporter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index a82d3d408..9c44b9eeb 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -65,7 +65,6 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: except Exception as e: logger.error("Error exporting spans to the database: %s", e) - print("Error exporting spans to the database: %s", e) return SpanExportResult.FAILURE return SpanExportResult.SUCCESS From fbc6e60935f20b0df28495501c957d2c56f8f453 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 17:46:21 -0500 Subject: [PATCH 30/59] updates --- examples/experimental/otel_exporter.ipynb | 169 +----------------- src/core/trulens/core/database/sqlalchemy.py | 15 +- .../otel_tracing/core/exporter.py | 1 + 3 files changed, 18 insertions(+), 167 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 34f15ecce..bae59e9d8 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -12,17 +12,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Adding /Users/gtokernliang/projects/trulens to sys.path\n" - ] - } - ], + "outputs": [], "source": [ "from pathlib import Path\n", "import sys\n", @@ -36,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -64,28 +56,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🦑 Initialized with db url snowflake://gtokernliang:***@fab02971/GTOK/PUBLIC?role=ENGINEER&warehouse=DKUROKAWA .\n", - "🔒 Secret keys will not be included in the database.\n", - "Set TruLens workspace version tag: [('Statement executed successfully.',)]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Updating app_name and app_version in apps table: 0it [00:00, ?it/s]\n", - "Updating app_id in records table: 0it [00:00, ?it/s]\n", - "Updating app_json in apps table: 0it [00:00, ?it/s]\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "\n", @@ -116,132 +89,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "skipping base because of class\n", - "skipping base because of class\n", - "decorating \n", - "decorating \n", - "decorating \n", - "{\n", - " \"name\": \"nested2\",\n", - " \"context\": {\n", - " \"trace_id\": \"0x4ed62c9e990bf71443c0bcedf0f719cc\",\n", - " \"span_id\": \"0x324e378d20841912\",\n", - " \"trace_state\": \"[]\"\n", - " },\n", - " \"kind\": \"SpanKind.INTERNAL\",\n", - " \"parent_id\": \"0xae573ae47e6ebc11\",\n", - " \"start_time\": \"2024-12-18T22:32:52.897168Z\",\n", - " \"end_time\": \"2024-12-18T22:32:52.897215Z\",\n", - " \"status\": {\n", - " \"status_code\": \"UNSET\"\n", - " },\n", - " \"attributes\": {\n", - " \"name\": \"nested2\",\n", - " \"kind\": \"SPAN_KIND_TRULENS\",\n", - " \"parent_span_id\": 3624895829355272466,\n", - " \"nested2_ret\": \"nested2: test\",\n", - " \"nested2_args[0]\": \"test\",\n", - " \"status\": \"STATUS_CODE_UNSET\"\n", - " },\n", - " \"events\": [],\n", - " \"links\": [],\n", - " \"resource\": {\n", - " \"attributes\": {\n", - " \"telemetry.sdk.language\": \"python\",\n", - " \"telemetry.sdk.name\": \"opentelemetry\",\n", - " \"telemetry.sdk.version\": \"1.25.0\",\n", - " \"service.name\": \"trulens\"\n", - " },\n", - " \"schema_url\": \"\"\n", - " }\n", - "}\n", - "Error exporting spans to the database: %s 1 validation error for Event\n", - "event_id\n", - " Field required [type=missing, input_value={'record': mappingproxy({...'12562574438621428753'}}, input_type=dict]\n", - " For further information visit https://errors.pydantic.dev/2.10/v/missing\n", - "{\n", - " \"name\": \"nested\",\n", - " \"context\": {\n", - " \"trace_id\": \"0x4ed62c9e990bf71443c0bcedf0f719cc\",\n", - " \"span_id\": \"0xae573ae47e6ebc11\",\n", - " \"trace_state\": \"[]\"\n", - " },\n", - " \"kind\": \"SpanKind.INTERNAL\",\n", - " \"parent_id\": \"0x7cfd5e50f405e5be\",\n", - " \"start_time\": \"2024-12-18T22:32:52.897083Z\",\n", - " \"end_time\": \"2024-12-18T22:32:52.897914Z\",\n", - " \"status\": {\n", - " \"status_code\": \"UNSET\"\n", - " },\n", - " \"attributes\": {\n", - " \"name\": \"nested\",\n", - " \"kind\": \"SPAN_KIND_TRULENS\",\n", - " \"parent_span_id\": 12562574438621428753,\n", - " \"nested_attr1\": \"value1\",\n", - " \"status\": \"STATUS_CODE_UNSET\"\n", - " },\n", - " \"events\": [],\n", - " \"links\": [],\n", - " \"resource\": {\n", - " \"attributes\": {\n", - " \"telemetry.sdk.language\": \"python\",\n", - " \"telemetry.sdk.name\": \"opentelemetry\",\n", - " \"telemetry.sdk.version\": \"1.25.0\",\n", - " \"service.name\": \"trulens\"\n", - " },\n", - " \"schema_url\": \"\"\n", - " }\n", - "}\n", - "Error exporting spans to the database: %s 1 validation error for Event\n", - "event_id\n", - " Field required [type=missing, input_value={'record': mappingproxy({... '9006458531595281854'}}, input_type=dict]\n", - " For further information visit https://errors.pydantic.dev/2.10/v/missing\n", - "{\n", - " \"name\": \"respond_to_query\",\n", - " \"context\": {\n", - " \"trace_id\": \"0x4ed62c9e990bf71443c0bcedf0f719cc\",\n", - " \"span_id\": \"0x7cfd5e50f405e5be\",\n", - " \"trace_state\": \"[]\"\n", - " },\n", - " \"kind\": \"SpanKind.INTERNAL\",\n", - " \"parent_id\": null,\n", - " \"start_time\": \"2024-12-18T22:32:52.896826Z\",\n", - " \"end_time\": \"2024-12-18T22:32:52.898300Z\",\n", - " \"status\": {\n", - " \"status_code\": \"UNSET\"\n", - " },\n", - " \"attributes\": {\n", - " \"name\": \"respond_to_query\",\n", - " \"kind\": \"SPAN_KIND_TRULENS\",\n", - " \"parent_span_id\": 9006458531595281854,\n", - " \"status\": \"STATUS_CODE_UNSET\"\n", - " },\n", - " \"events\": [],\n", - " \"links\": [],\n", - " \"resource\": {\n", - " \"attributes\": {\n", - " \"telemetry.sdk.language\": \"python\",\n", - " \"telemetry.sdk.name\": \"opentelemetry\",\n", - " \"telemetry.sdk.version\": \"1.25.0\",\n", - " \"service.name\": \"trulens\"\n", - " },\n", - " \"schema_url\": \"\"\n", - " }\n", - "}\n", - "Error exporting spans to the database: %s 1 validation error for Event\n", - "event_id\n", - " Field required [type=missing, input_value={'record': mappingproxy({...4540', 'parent_id': ''}}, input_type=dict]\n", - " For further information visit https://errors.pydantic.dev/2.10/v/missing\n" - ] - } - ], + "outputs": [], "source": [ "from trulens.apps.custom import TruCustomApp\n", "\n", @@ -268,8 +118,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.11" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/src/core/trulens/core/database/sqlalchemy.py b/src/core/trulens/core/database/sqlalchemy.py index 5544707dd..d604508de 100644 --- a/src/core/trulens/core/database/sqlalchemy.py +++ b/src/core/trulens/core/database/sqlalchemy.py @@ -44,7 +44,6 @@ from trulens.core.schema import app as app_schema from trulens.core.schema import base as base_schema from trulens.core.schema import dataset as dataset_schema -from trulens.core.schema import event as event_schema from trulens.core.schema import feedback as feedback_schema from trulens.core.schema import groundtruth as groundtruth_schema from trulens.core.schema import record as record_schema @@ -74,18 +73,20 @@ def patch_insert(statement: Insert, compiler: SQLCompiler, **kw): if statement.table.name.endswith("_events"): insert_statement = insert_statement.replace( - "VALUES (%(record)s, %(record_attributes)s, %(record_type)s, %(resource_attributes)s, %(start_timestamp)s, %(timestamp)s, %(trace)s)", + "VALUES (%(record)s, %(event_id)s, %(record_attributes)s, %(record_type)s, %(resource_attributes)s, %(start_timestamp)s, %(timestamp)s, %(trace)s)", """ SELECT PARSE_JSON(column1), - PARSE_JSON(column2), - column3, - PARSE_JSON(column4), - column5, + column2, + PARSE_JSON(column3), + column4, + PARSE_JSON(column5), column6, - PARSE_JSON(column7), + column7, + PARSE_JSON(column8), from VALUES ( %(record)s, + %(event_id)s, %(record_attributes)s, %(record_type)s, %(resource_attributes)s, diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index 9c44b9eeb..a82d3d408 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -65,6 +65,7 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: except Exception as e: logger.error("Error exporting spans to the database: %s", e) + print("Error exporting spans to the database: %s", e) return SpanExportResult.FAILURE return SpanExportResult.SUCCESS From efa8c6e31ffbdbd62827bb3dda453265e4e2a95c Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 18:07:56 -0500 Subject: [PATCH 31/59] update a bit --- src/core/trulens/experimental/otel_tracing/core/instrument.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index acc9d76e0..4ab7bb230 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -54,9 +54,9 @@ def wrapper(*args, **kwargs): "parent_span_id", parent_span.get_span_context().span_id ) - try: - ret = custom_instrument(func)(*args, **kwargs) + ret = custom_instrument(func)(*args, **kwargs) + try: attributes_to_add = {} # Set the user provider attributes. From cf97904ae9da7c2a8ce670005d5bd48290d13913 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Wed, 18 Dec 2024 18:35:15 -0500 Subject: [PATCH 32/59] remove redundant print --- src/core/trulens/experimental/otel_tracing/core/exporter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/exporter.py b/src/core/trulens/experimental/otel_tracing/core/exporter.py index a82d3d408..9c44b9eeb 100644 --- a/src/core/trulens/experimental/otel_tracing/core/exporter.py +++ b/src/core/trulens/experimental/otel_tracing/core/exporter.py @@ -65,7 +65,6 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: except Exception as e: logger.error("Error exporting spans to the database: %s", e) - print("Error exporting spans to the database: %s", e) return SpanExportResult.FAILURE return SpanExportResult.SUCCESS From dc98c62f465f6bb93e411b4593bc25326e6a2cc8 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 19 Dec 2024 15:20:52 -0500 Subject: [PATCH 33/59] remove snowflake --- examples/experimental/otel_exporter.ipynb | 99 +++++++++++++++++++- src/core/trulens/core/database/sqlalchemy.py | 47 ---------- 2 files changed, 97 insertions(+), 49 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index bae59e9d8..b7e6c9454 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -121,6 +121,101 @@ "pygments_lexer": "ipython3" } }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Add base dir to path to be able to access test folder.\n", + "base_dir = Path().cwd().parent.parent.resolve()\n", + "if str(base_dir) not in sys.path:\n", + " print(f\"Adding {base_dir} to sys.path\")\n", + " sys.path.append(str(base_dir))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.experimental.otel_tracing.core.instrument import instrument\n", + "\n", + "\n", + "class TestApp:\n", + " @instrument()\n", + " def respond_to_query(self, query: str) -> str:\n", + " return f\"answer: {self.nested(query)}\"\n", + "\n", + " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", + " def nested(self, query: str) -> str:\n", + " return f\"nested: {self.nested2(query)}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, *args, **kwargs: {\n", + " \"nested2_ret\": ret,\n", + " \"nested2_args[0]\": args[0],\n", + " }\n", + " )\n", + " def nested2(self, query: str) -> str:\n", + " return f\"nested2: {query}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import dotenv\n", + "from trulens.core.session import TruSession\n", + "from trulens.experimental.otel_tracing.core.init import init\n", + "\n", + "dotenv.load_dotenv()\n", + "\n", + "session = TruSession()\n", + "session.reset_database()\n", + "init(session, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.apps.custom import TruCustomApp\n", + "\n", + "test_app = TestApp()\n", + "custom_app = TruCustomApp(test_app)\n", + "\n", + "with custom_app as recording:\n", + " test_app.respond_to_query(\"test\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "trulens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/src/core/trulens/core/database/sqlalchemy.py b/src/core/trulens/core/database/sqlalchemy.py index d604508de..de84616d6 100644 --- a/src/core/trulens/core/database/sqlalchemy.py +++ b/src/core/trulens/core/database/sqlalchemy.py @@ -25,15 +25,11 @@ import pandas as pd import pydantic from pydantic import Field -from snowflake.sqlalchemy import dialect as SnowflakeDialect import sqlalchemy as sa from sqlalchemy import Table -from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import joinedload from sqlalchemy.orm import sessionmaker from sqlalchemy.sql import text as sql_text -from sqlalchemy.sql.compiler import SQLCompiler -from sqlalchemy.sql.expression import Insert from trulens.core.database import base as core_db from trulens.core.database import exceptions as db_exceptions from trulens.core.database import migrations as db_migrations @@ -57,49 +53,6 @@ logger = logging.getLogger(__name__) -@compiles(Insert, SnowflakeDialect.name) -def patch_insert(statement: Insert, compiler: SQLCompiler, **kw): - """ - Patches INSERT SQL queries so sqlalchemy ORM will support Snowflake OBJECT. - - See: - * https://github.com/snowflakedb/snowflake-sqlalchemy/issues/299 - * https://github.com/snowflakedb/snowflake-sqlalchemy/issues/411 - - For more information (e.g. read about the parameters), please look at: - * https://docs.sqlalchemy.org/en/20/core/compiler.html - """ - insert_statement = compiler.visit_insert(statement, **kw) - - if statement.table.name.endswith("_events"): - insert_statement = insert_statement.replace( - "VALUES (%(record)s, %(event_id)s, %(record_attributes)s, %(record_type)s, %(resource_attributes)s, %(start_timestamp)s, %(timestamp)s, %(trace)s)", - """ -SELECT - PARSE_JSON(column1), - column2, - PARSE_JSON(column3), - column4, - PARSE_JSON(column5), - column6, - column7, - PARSE_JSON(column8), -from VALUES ( - %(record)s, - %(event_id)s, - %(record_attributes)s, - %(record_type)s, - %(resource_attributes)s, - %(start_timestamp)s, - %(timestamp)s, - %(trace)s -) -""", - ) - - return insert_statement - - class SnowflakeImpl(DefaultImpl): __dialect__ = "snowflake" From 7dae6ff7d24ed3d38cf921b9b3f99d5de217d2e5 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 19 Dec 2024 15:32:36 -0500 Subject: [PATCH 34/59] remove instrument --- .../trulens/experimental/otel_tracing/core/instrument.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 4ab7bb230..1c54cfb52 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -3,7 +3,6 @@ from typing import Any, Callable, Optional, Union from opentelemetry import trace -from trulens.apps.custom import instrument as custom_instrument from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME from trulens.experimental.otel_tracing.core.semantic import ( TRULENS_SELECTOR_NAME, @@ -46,7 +45,6 @@ def wrapper(*args, **kwargs): ) ) as parent_span: span = trace.get_current_span() - _instrumented_object, *rest_args = args span.set_attribute("name", func.__name__) span.set_attribute("kind", "SPAN_KIND_TRULENS") @@ -54,7 +52,7 @@ def wrapper(*args, **kwargs): "parent_span_id", parent_span.get_span_context().span_id ) - ret = custom_instrument(func)(*args, **kwargs) + ret = func(*args, **kwargs) try: attributes_to_add = {} @@ -62,9 +60,7 @@ def wrapper(*args, **kwargs): # Set the user provider attributes. if attributes: if callable(attributes): - attributes_to_add = attributes( - ret, *rest_args, **kwargs - ) + attributes_to_add = attributes(ret, *args, **kwargs) else: attributes_to_add = attributes From e1b685cdb1cded46252bab484fb25c0cce542aa3 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 19:26:36 -0500 Subject: [PATCH 35/59] prepend namespace --- examples/experimental/otel_exporter.ipynb | 181 ++++++------------ .../otel_tracing/core/instrument.py | 101 +++++++--- .../otel_tracing/core/semantic.py | 1 - .../semconv/trulens/otel/semconv/trace.py | 16 +- 4 files changed, 146 insertions(+), 153 deletions(-) delete mode 100644 src/core/trulens/experimental/otel_tracing/core/semantic.py diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index b7e6c9454..a3a145b3e 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -1,125 +1,14 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install opentelemetry-api\n", - "# !pip install opentelemetry-sdk" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import sys\n", - "\n", - "# Add base dir to path to be able to access test folder.\n", - "base_dir = Path().cwd().parent.parent.resolve()\n", - "if str(base_dir) not in sys.path:\n", - " print(f\"Adding {base_dir} to sys.path\")\n", - " sys.path.append(str(base_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.experimental.otel_tracing.core.instrument import instrument\n", - "\n", - "\n", - "class TestApp:\n", - " @instrument()\n", - " def respond_to_query(self, query: str) -> str:\n", - " return f\"answer: {self.nested(query)}\"\n", - "\n", - " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", - " def nested(self, query: str) -> str:\n", - " return f\"nested: {self.nested2(query)}\"\n", - "\n", - " @instrument(\n", - " attributes=lambda ret, *args, **kwargs: {\n", - " \"nested2_ret\": ret,\n", - " \"nested2_args[0]\": args[0],\n", - " }\n", - " )\n", - " def nested2(self, query: str) -> str:\n", - " return f\"nested2: {query}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "import dotenv\n", - "from trulens.connectors.snowflake import SnowflakeConnector\n", - "from trulens.core.session import TruSession\n", - "from trulens.experimental.otel_tracing.core.init import init\n", - "\n", - "dotenv.load_dotenv()\n", - "\n", - "connection_params = {\n", - " \"account\": os.environ[\"SNOWFLAKE_ACCOUNT\"],\n", - " \"user\": os.environ[\"SNOWFLAKE_USER\"],\n", - " \"password\": os.environ[\"SNOWFLAKE_USER_PASSWORD\"],\n", - " \"database\": os.environ[\"SNOWFLAKE_DATABASE\"],\n", - " \"schema\": os.environ[\"SNOWFLAKE_SCHEMA\"],\n", - " \"warehouse\": os.environ[\"SNOWFLAKE_WAREHOUSE\"],\n", - " \"role\": os.environ[\"SNOWFLAKE_ROLE\"],\n", - "}\n", - "\n", - "connector = SnowflakeConnector(\n", - " **connection_params, database_redact_keys=True, database_args=None\n", - ")\n", - "session = TruSession(connector=connector)\n", - "session.reset_database()\n", - "init(session, debug=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.apps.custom import TruCustomApp\n", - "\n", - "test_app = TestApp()\n", - "custom_app = TruCustomApp(test_app)\n", - "\n", - "with custom_app as recording:\n", - " test_app.respond_to_query(\"test\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "trulens", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install opentelemetry-api\n", + "# !pip install opentelemetry-sdk" + ] }, { "cell_type": "code", @@ -137,6 +26,26 @@ " sys.path.append(str(base_dir))" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "root = logging.getLogger()\n", + "root.setLevel(logging.DEBUG)\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setLevel(logging.DEBUG)\n", + "handler.addFilter(logging.Filter(\"trulens\"))\n", + "formatter = logging.Formatter(\n", + " \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n", + ")\n", + "handler.setFormatter(formatter)\n", + "root.addHandler(handler)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -156,13 +65,33 @@ " return f\"nested: {self.nested2(query)}\"\n", "\n", " @instrument(\n", - " attributes=lambda ret, *args, **kwargs: {\n", + " attributes=lambda ret, exception, *args, **kwargs: {\n", " \"nested2_ret\": ret,\n", " \"nested2_args[0]\": args[0],\n", " }\n", " )\n", " def nested2(self, query: str) -> str:\n", - " return f\"nested2: {query}\"" + " nested_result = \"\"\n", + "\n", + " try:\n", + " nested_result = self.nested3(query)\n", + " except Exception:\n", + " pass\n", + "\n", + " return f\"nested2: {nested_result}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, exception, *args, **kwargs: {\n", + " \"nested3_ex\": exception.args if exception else None,\n", + " \"nested3_ret\": ret,\n", + " \"selector_name\": \"special\",\n", + " \"cows\": \"moo\",\n", + " }\n", + " )\n", + " def nested3(self, query: str) -> str:\n", + " if query == \"throw\":\n", + " raise ValueError(\"nested3 exception\")\n", + " return \"nested3\"" ] }, { @@ -178,6 +107,7 @@ "dotenv.load_dotenv()\n", "\n", "session = TruSession()\n", + "session.experimental_enable_feature(\"otel_tracing\")\n", "session.reset_database()\n", "init(session, debug=True)" ] @@ -194,7 +124,10 @@ "custom_app = TruCustomApp(test_app)\n", "\n", "with custom_app as recording:\n", - " test_app.respond_to_query(\"test\")" + " test_app.respond_to_query(\"test\")\n", + "\n", + "with custom_app as recording:\n", + " test_app.respond_to_query(\"throw\")" ] } ], diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 1c54cfb52..d9e17c7e8 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -4,16 +4,19 @@ from opentelemetry import trace from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME -from trulens.experimental.otel_tracing.core.semantic import ( - TRULENS_SELECTOR_NAME, -) +from trulens.otel.semconv.trace import SpanAttributes logger = logging.getLogger(__name__) def instrument( attributes: Optional[ - Union[dict[str, Any], Callable[[Any, Any, Any], dict[str, Any]]] + Union[ + dict[str, Any], + Callable[ + [Optional[Any], Optional[Exception], Any, Any], dict[str, Any] + ], + ] ] = {}, ): """ @@ -21,16 +24,35 @@ def instrument( wrapped by TruCustomApp, with OpenTelemetry tracing. """ - def _validate_selector_name(final_attributes: dict[str, Any]): - if TRULENS_SELECTOR_NAME in final_attributes: - selector_name = final_attributes[TRULENS_SELECTOR_NAME] + def _validate_selector_name(attributes: dict[str, Any]) -> dict[str, Any]: + result = attributes.copy() + + if ( + SpanAttributes.SELECTOR_NAME_KEY in result + and SpanAttributes.SELECTOR_NAME in result + ): + raise ValueError( + f"Both {SpanAttributes.SELECTOR_NAME_KEY} and {SpanAttributes.SELECTOR_NAME} cannot be set." + ) + + if SpanAttributes.SELECTOR_NAME in result: + # Transfer the trulens namespaced to the non-trulens namespaced key. + result[SpanAttributes.SELECTOR_NAME_KEY] = result[ + SpanAttributes.SELECTOR_NAME + ] + del result[SpanAttributes.SELECTOR_NAME] + + if SpanAttributes.SELECTOR_NAME_KEY in result: + selector_name = result[SpanAttributes.SELECTOR_NAME_KEY] if not isinstance(selector_name, str): raise ValueError( f"Selector name must be a string, not {type(selector_name)}" ) - def _validate_attributes(final_attributes: dict[str, Any]): - _validate_selector_name(final_attributes) + return result + + def _validate_attributes(attributes: dict[str, Any]) -> dict[str, Any]: + return _validate_selector_name(attributes) # TODO: validate OTEL attributes. # TODO: validate span type attributes. @@ -43,39 +65,64 @@ def wrapper(*args, **kwargs): .start_as_current_span( name=func.__name__, ) - ) as parent_span: - span = trace.get_current_span() - - span.set_attribute("name", func.__name__) - span.set_attribute("kind", "SPAN_KIND_TRULENS") - span.set_attribute( - "parent_span_id", parent_span.get_span_context().span_id - ) + ) as span: + ret = None + exception: Optional[Exception] = None - ret = func(*args, **kwargs) + try: + ret = func(*args, **kwargs) + except Exception as e: + # We want to get into the next clause to allow the users to still add attributes. + # It's on the user to deal with None as a return value. + exception = e try: attributes_to_add = {} + # Since we're decoratoring a method in a trulens app, the first argument is self, + # which we should ignore. + _self, *rest = args + # Set the user provider attributes. if attributes: if callable(attributes): - attributes_to_add = attributes(ret, *args, **kwargs) + attributes_to_add = attributes( + ret, exception, *rest, **kwargs + ) else: attributes_to_add = attributes logger.info(f"Attributes to add: {attributes_to_add}") - _validate_attributes(attributes_to_add) - - for key, value in attributes_to_add.items(): - span.set_attribute(key, value) - - except Exception: - span.set_attribute("status", "STATUS_CODE_ERROR") + final_attributes = _validate_attributes(attributes_to_add) + + prefix = "trulens." + if ( + SpanAttributes.SPAN_TYPE in final_attributes + and final_attributes[SpanAttributes.SPAN_TYPE] + != SpanAttributes.SpanType.UNKNOWN + ): + prefix += ( + final_attributes[SpanAttributes.SPAN_TYPE] + "." + ) + + for key, value in final_attributes.items(): + span.set_attribute(prefix + key, value) + + if ( + key != SpanAttributes.SELECTOR_NAME_KEY + and SpanAttributes.SELECTOR_NAME_KEY + in final_attributes + ): + span.set_attribute( + f"trulens.{final_attributes[SpanAttributes.SELECTOR_NAME_KEY]}.{key}", + value, + ) + + except Exception as e: + logger.error(f"Error setting attributes: {e}") return None - span.set_attribute("status", "STATUS_CODE_UNSET") return ret return wrapper diff --git a/src/core/trulens/experimental/otel_tracing/core/semantic.py b/src/core/trulens/experimental/otel_tracing/core/semantic.py deleted file mode 100644 index d66bd00b8..000000000 --- a/src/core/trulens/experimental/otel_tracing/core/semantic.py +++ /dev/null @@ -1 +0,0 @@ -TRULENS_SELECTOR_NAME = "trulens.selector_name" diff --git a/src/otel/semconv/trulens/otel/semconv/trace.py b/src/otel/semconv/trulens/otel/semconv/trace.py index 5b55f0bca..4e2e436ac 100644 --- a/src/otel/semconv/trulens/otel/semconv/trace.py +++ b/src/otel/semconv/trulens/otel/semconv/trace.py @@ -34,7 +34,21 @@ class SpanAttributes: In some cases below, we also include span name or span name prefix. """ - SPAN_TYPES = "trulens.span_types" + BASE = "trulens." + """ + Base prefix for the other keys. + """ + + SPAN_TYPE = BASE + "span_type" + + SPAN_TYPES = BASE + "span_types" + + SELECTOR_NAME_KEY = "selector_name" + + SELECTOR_NAME = BASE + "selector_name" + """ + User-defined selector name for the current span. + """ class SpanType(str, Enum): """Span type attribute values. From bf80a11c512fd7e9c79fc0a16a98dee2ee3fe97e Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 19:29:43 -0500 Subject: [PATCH 36/59] update semcov --- src/otel/semconv/trulens/otel/semconv/trace.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/otel/semconv/trulens/otel/semconv/trace.py b/src/otel/semconv/trulens/otel/semconv/trace.py index 4e2e436ac..9050f81ae 100644 --- a/src/otel/semconv/trulens/otel/semconv/trace.py +++ b/src/otel/semconv/trulens/otel/semconv/trace.py @@ -40,12 +40,18 @@ class SpanAttributes: """ SPAN_TYPE = BASE + "span_type" - - SPAN_TYPES = BASE + "span_types" + """ + Span type attribute. + """ SELECTOR_NAME_KEY = "selector_name" + """ + Key for the user-defined selector name for the current span. + Here to help us check both trulens.selector_name and selector_name + to verify the user attributes and make corrections if necessary. + """ - SELECTOR_NAME = BASE + "selector_name" + SELECTOR_NAME = BASE + SELECTOR_NAME_KEY """ User-defined selector name for the current span. """ From 79b157f167e6d0f5ceec660312e17f0cbf4629a8 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sat, 21 Dec 2024 12:24:41 -0800 Subject: [PATCH 37/59] Fix api tests. --- src/core/trulens/core/database/base.py | 9 --------- src/core/trulens/core/database/sqlalchemy.py | 3 --- 2 files changed, 12 deletions(-) diff --git a/src/core/trulens/core/database/base.py b/src/core/trulens/core/database/base.py index 8bd256919..8501fa652 100644 --- a/src/core/trulens/core/database/base.py +++ b/src/core/trulens/core/database/base.py @@ -91,15 +91,6 @@ def check_db_revision(self): """ raise NotImplementedError() - @abc.abstractmethod - def get_db_dialect(self) -> Optional[str]: - """Get the dialect of the database. - - Returns: - The dialect of the database. - """ - raise NotImplementedError() - @abc.abstractmethod def get_db_revision(self) -> Optional[str]: """Get the current revision of the database. diff --git a/src/core/trulens/core/database/sqlalchemy.py b/src/core/trulens/core/database/sqlalchemy.py index c03ccd6ad..d0923a731 100644 --- a/src/core/trulens/core/database/sqlalchemy.py +++ b/src/core/trulens/core/database/sqlalchemy.py @@ -252,9 +252,6 @@ def check_db_revision(self): db_utils.check_db_revision(self.engine, self.table_prefix) - def get_db_dialect(self) -> Optional[str]: - return self.engine.dialect.name if self.engine else None - def get_db_revision(self) -> Optional[str]: if self.engine is None: raise ValueError("Database engine not initialized.") From 0e3564bde600357e86e03e1646c40427886f9b29 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sat, 21 Dec 2024 12:45:00 -0800 Subject: [PATCH 38/59] Incorporate my own comments in the review except for the adding of a test. --- .../otel_tracing/core/instrument.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index d9e17c7e8..82d46b627 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -10,6 +10,7 @@ def instrument( + *, attributes: Optional[ Union[ dict[str, Any], @@ -52,6 +53,12 @@ def _validate_selector_name(attributes: dict[str, Any]) -> dict[str, Any]: return result def _validate_attributes(attributes: dict[str, Any]) -> dict[str, Any]: + if not isinstance(attributes, dict) or any([ + not isinstance(key, str) for key in attributes.keys() + ]): + raise ValueError( + "Attributes must be a dictionary with string keys." + ) return _validate_selector_name(attributes) # TODO: validate OTEL attributes. # TODO: validate span type attributes. @@ -67,27 +74,28 @@ def wrapper(*args, **kwargs): ) ) as span: ret = None - exception: Optional[Exception] = None + func_exception: Optional[Exception] = None + attributes_exception: Optional[Exception] = None try: ret = func(*args, **kwargs) except Exception as e: # We want to get into the next clause to allow the users to still add attributes. # It's on the user to deal with None as a return value. - exception = e + func_exception = e try: attributes_to_add = {} - # Since we're decoratoring a method in a trulens app, the first argument is self, + # Since we're decorating a method in a trulens app, the first argument is self, # which we should ignore. - _self, *rest = args + args = args[1:] # Set the user provider attributes. if attributes: if callable(attributes): attributes_to_add = attributes( - ret, exception, *rest, **kwargs + ret, func_exception, *args, **kwargs ) else: attributes_to_add = attributes @@ -120,8 +128,13 @@ def wrapper(*args, **kwargs): ) except Exception as e: + attributes_exception = e logger.error(f"Error setting attributes: {e}") - return None + + if func_exception: + raise func_exception + if attributes_exception: + raise attributes_exception return ret From 45282584275f6950d5af0ceaf5d55e5a1d9dc52e Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sat, 21 Dec 2024 12:56:37 -0800 Subject: [PATCH 39/59] Add framework for test. --- tests/unit/test_otel_instrument.py | 76 ++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/unit/test_otel_instrument.py diff --git a/tests/unit/test_otel_instrument.py b/tests/unit/test_otel_instrument.py new file mode 100644 index 000000000..183eb1168 --- /dev/null +++ b/tests/unit/test_otel_instrument.py @@ -0,0 +1,76 @@ +""" +Tests for OTEL instrument decorator. +""" + +from unittest import TestCase +from unittest import main + +from trulens.apps.custom import TruCustomApp +from trulens.core.session import TruSession +from trulens.experimental.otel_tracing.core.init import init +from trulens.experimental.otel_tracing.core.instrument import instrument + + +class _TestApp: + @instrument() + def respond_to_query(self, query: str) -> str: + return f"answer: {self.nested(query)}" + + @instrument(attributes={"nested_attr1": "value1"}) + def nested(self, query: str) -> str: + return f"nested: {self.nested2(query)}" + + @instrument( + attributes=lambda ret, exception, *args, **kwargs: { + "nested2_ret": ret, + "nested2_args[0]": args[0], + } + ) + def nested2(self, query: str) -> str: + nested_result = "" + + try: + nested_result = self.nested3(query) + except Exception: + pass + + return f"nested2: {nested_result}" + + @instrument( + attributes=lambda ret, exception, *args, **kwargs: { + "nested3_ex": exception.args if exception else None, + "nested3_ret": ret, + "selector_name": "special", + "cows": "moo", + } + ) + def nested3(self, query: str) -> str: + if query == "throw": + raise ValueError("nested3 exception") + return "nested3" + + +class TestOtelInstrument(TestCase): + def setUp(self): + pass + + def test_deterministic_app_id(self): + session = TruSession() + session.experimental_enable_feature("otel_tracing") + session.reset_database() + init(session, debug=True) + + test_app = _TestApp() + custom_app = TruCustomApp(test_app) + + with custom_app as recording: + test_app.respond_to_query("test") + + with custom_app as recording: + test_app.respond_to_query("throw") + + print(recording) + + +if __name__ == "__main__": + main() From a1c461c2461c2ce6090c123bb4d829fdbe91f108 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sat, 21 Dec 2024 23:17:12 -0800 Subject: [PATCH 40/59] Add in better test. --- tests/test.py | 17 +- ..._instrument__test_deterministic_app_id.csv | 9 ++ tests/unit/test_otel_instrument.py | 153 ++++++++++++++++-- 3 files changed, 164 insertions(+), 15 deletions(-) create mode 100644 tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv diff --git a/tests/test.py b/tests/test.py index 729afa099..7a06a8715 100644 --- a/tests/test.py +++ b/tests/test.py @@ -27,6 +27,7 @@ import unittest from unittest import TestCase +import pandas as pd import pydantic from pydantic import BaseModel from trulens.core._utils.pycompat import ReferenceType @@ -225,7 +226,9 @@ class WithJSONTestCase(TestCase): """TestCase mixin class that adds JSON comparisons and golden expectation handling.""" - def load_golden(self, golden_path: Union[str, Path]) -> serial_utils.JSON: + def load_golden( + self, golden_path: Union[str, Path] + ) -> Union[serial_utils.JSON, pd.DataFrame]: """Load the golden file `path` and return its contents. Args: @@ -240,6 +243,10 @@ def load_golden(self, golden_path: Union[str, Path]) -> serial_utils.JSON: loader = functools.partial(json.load) elif ".yaml" in golden_path.suffixes or ".yml" in golden_path.suffixes: loader = functools.partial(yaml.load, Loader=yaml.FullLoader) + elif ".csv" in golden_path.suffixes: + loader = functools.partial(pd.read_csv, index_col=0) + elif ".parquet" in golden_path.suffixes: + loader = functools.partial(pd.read_parquet, index_col=0) else: raise ValueError(f"Unknown file extension {golden_path}.") @@ -250,7 +257,9 @@ def load_golden(self, golden_path: Union[str, Path]) -> serial_utils.JSON: return loader(f) def write_golden( - self, golden_path: Union[str, Path], data: serial_utils.JSON + self, + golden_path: Union[str, Path], + data: Union[serial_utils.JSON, pd.DataFrame], ) -> None: """If writing golden file is enabled, write the golden file `path` with `data` and raise exception indicating so. @@ -272,6 +281,10 @@ def write_golden( writer = functools.partial(json.dump, indent=2, sort_keys=True) elif golden_path.suffix == ".yaml": writer = functools.partial(yaml.dump, sort_keys=True) + elif golden_path.suffix == ".csv": + writer = lambda data, f: data.to_csv(f) + elif golden_path.suffix == ".parquet": + writer = lambda data, f: data.to_parquet(f) else: raise ValueError(f"Unknown file extension {golden_path.suffix}.") diff --git a/tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv b/tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv new file mode 100644 index 000000000..f03fabb32 --- /dev/null +++ b/tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv @@ -0,0 +1,9 @@ +,record,event_id,record_attributes,record_type,resource_attributes,start_timestamp,timestamp,trace +0,"{'name': 'respond_to_query', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '', 'status': 'STATUS_CODE_UNSET'}",11560964145073308497,{},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817734,2024-12-21 22:46:48.822087,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '', 'span_id': '11560964145073308497'}" +1,"{'name': 'nested', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '11560964145073308497', 'status': 'STATUS_CODE_UNSET'}",956390419060041970,{'trulens.nested_attr1': 'value1'},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817764,2024-12-21 22:46:48.821018,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '11560964145073308497', 'span_id': '956390419060041970'}" +2,"{'name': 'nested2', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '956390419060041970', 'status': 'STATUS_CODE_UNSET'}",7191373700275539380,"{'trulens.nested2_ret': 'nested2: nested3', 'trulens.nested2_args[0]': 'test'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817784,2024-12-21 22:46:48.819862,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '956390419060041970', 'span_id': '7191373700275539380'}" +3,"{'name': 'nested3', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '7191373700275539380', 'status': 'STATUS_CODE_UNSET'}",11561410003717533750,"{'trulens.nested3_ret': 'nested3', 'trulens.special.nested3_ret': 'nested3', 'trulens.selector_name': 'special', 'trulens.cows': 'moo', 'trulens.special.cows': 'moo'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817801,2024-12-21 22:46:48.817858,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '7191373700275539380', 'span_id': '11561410003717533750'}" +4,"{'name': 'respond_to_query', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '', 'status': 'STATUS_CODE_UNSET'}",15712161957924150536,{},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823641,2024-12-21 22:46:48.827118,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '', 'span_id': '15712161957924150536'}" +5,"{'name': 'nested', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '15712161957924150536', 'status': 'STATUS_CODE_UNSET'}",14875482935475817656,{'trulens.nested_attr1': 'value1'},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823666,2024-12-21 22:46:48.826205,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '15712161957924150536', 'span_id': '14875482935475817656'}" +6,"{'name': 'nested2', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '14875482935475817656', 'status': 'STATUS_CODE_UNSET'}",7500502012444675242,"{'trulens.nested2_ret': 'nested2: ', 'trulens.nested2_args[0]': 'throw'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823683,2024-12-21 22:46:48.825310,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '14875482935475817656', 'span_id': '7500502012444675242'}" +7,"{'name': 'nested3', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '7500502012444675242', 'status': 'STATUS_CODE_ERROR'}",9125451447180209633,"{'trulens.nested3_ex': ['nested3 exception'], 'trulens.special.nested3_ex': ['nested3 exception'], 'trulens.selector_name': 'special', 'trulens.cows': 'moo', 'trulens.special.cows': 'moo'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823710,2024-12-21 22:46:48.824262,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '7500502012444675242', 'span_id': '9125451447180209633'}" diff --git a/tests/unit/test_otel_instrument.py b/tests/unit/test_otel_instrument.py index 183eb1168..65956d655 100644 --- a/tests/unit/test_otel_instrument.py +++ b/tests/unit/test_otel_instrument.py @@ -2,14 +2,18 @@ Tests for OTEL instrument decorator. """ -from unittest import TestCase +from typing import Any, Dict from unittest import main +import pandas as pd +import sqlalchemy as sa from trulens.apps.custom import TruCustomApp from trulens.core.session import TruSession from trulens.experimental.otel_tracing.core.init import init from trulens.experimental.otel_tracing.core.instrument import instrument +from tests.test import TruTestCase + class _TestApp: @instrument() @@ -50,26 +54,149 @@ def nested3(self, query: str) -> str: return "nested3" -class TestOtelInstrument(TestCase): +class TestOtelInstrument(TruTestCase): def setUp(self): pass - def test_deterministic_app_id(self): - session = TruSession() - session.experimental_enable_feature("otel_tracing") - session.reset_database() - init(session, debug=True) + @staticmethod + def _get_events() -> pd.DataFrame: + tru_session = TruSession() + db = tru_session.connector.db + with db.session.begin() as db_session: + q = sa.select(db.orm.Event).order_by(db.orm.Event.start_timestamp) + return pd.read_sql(q, db_session.bind) + + @staticmethod + def _convert_column_types(df: pd.DataFrame): + df["event_id"] = df["event_id"].apply(str) + df["record_type"] = df["record_type"].apply(lambda x: eval(x)) + df["start_timestamp"] = df["start_timestamp"].apply(pd.Timestamp) + df["timestamp"] = df["timestamp"].apply(pd.Timestamp) + for json_column in [ + "record", + "record_attributes", + "resource_attributes", + "trace", + ]: + df[json_column] = df[json_column].apply(lambda x: eval(x)) + + def _compare_dfs_accounting_for_ids_and_timestamps( + self, expected: pd.DataFrame, actual: pd.DataFrame + ): + """ + Compare two Dataframes are equal, accounting for ids and timestamps. + That is: + 1. The ids between the two Dataframes may be different, but they have + to be consistent. That is, if one Dataframe reuses an id in two + places, then the other must as well. + 2. The timestamps between the two Dataframes may be different, but + they have to be in the same order. + + Args: + expected: expected results + actual: actual results + """ + id_mapping: Dict[str, str] = {} + timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp] = {} + self.assertEqual(len(expected), len(actual)) + self.assertListEqual(list(expected.columns), list(actual.columns)) + for i in range(len(expected)): + for col in expected.columns: + self._compare_entity( + expected.iloc[i][col], + actual.iloc[i][col], + id_mapping, + timestamp_mapping, + is_id=col.endswith("_id"), + locator=f"df.iloc[{i}][{col}]", + ) + # Ensure that the id mapping is a bijection. + self.assertEqual( + len(set(id_mapping.values())), + len(id_mapping), + "Ids are not a bijection!", + ) + # Ensure that the timestamp mapping is monotonic. + prev_value = None + for curr in sorted(timestamp_mapping.keys()): + if prev_value is not None: + self.assertLess( + prev_value, + timestamp_mapping[curr], + "Timestamps are not in the same order!", + ) + prev_value = timestamp_mapping[curr] + + def _compare_entity( + self, + expected: Any, + actual: Any, + id_mapping: Dict[str, str], + timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp], + is_id: bool, + locator: str, + ): + self.assertEqual( + type(expected), type(actual), f"Types of {locator} do not match!" + ) + if is_id: + self.assertEqual( + type(expected), str, f"Type of id {locator} is not a string!" + ) + if expected not in id_mapping: + id_mapping[expected] = actual + self.assertEqual( + id_mapping[expected], + actual, + f"Ids of {locator} are not consistent!", + ) + elif isinstance(expected, dict): + self.assertEqual( + expected.keys(), + actual.keys(), + f"Keys of {locator} do not match!", + ) + for k in expected.keys(): + self._compare_entity( + expected[k], + actual[k], + id_mapping, + timestamp_mapping, + is_id=k.endswith("_id"), + locator=f"{locator}[k]", + ) + elif isinstance(expected, pd.Timestamp): + if expected not in timestamp_mapping: + timestamp_mapping[expected] = actual + self.assertEqual( + timestamp_mapping[expected], + actual, + f"Timestamps of {locator} are not consistent!", + ) + else: + self.assertEqual(expected, actual, f"{locator} does not match!") + def test_deterministic_app_id(self): + # Set up. + tru_session = TruSession() + tru_session.experimental_enable_feature("otel_tracing") + tru_session.reset_database() + init(tru_session, debug=True) + # Create and run app. test_app = _TestApp() custom_app = TruCustomApp(test_app) - - with custom_app as recording: + with custom_app: test_app.respond_to_query("test") - - with custom_app as recording: + with custom_app: test_app.respond_to_query("throw") - - print(recording) + # Compare results to expected. + GOLDEN_FILENAME = "tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv" + actual = self._get_events() + self.assertEqual(len(actual), 8) + self.write_golden(GOLDEN_FILENAME, actual) + expected = self.load_golden(GOLDEN_FILENAME) + self._convert_column_types(expected) + self._compare_dfs_accounting_for_ids_and_timestamps(expected, actual) if __name__ == "__main__": From 1c34cd88a612671d8d8e82fe9fd41f2a3402ded9 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sun, 22 Dec 2024 10:55:49 -0800 Subject: [PATCH 41/59] Clean up some issues and fix import issues. --- Makefile | 11 +- .../otel_tracing/core/instrument.py | 4 - tests/test.py | 6 +- ...instrument__test_instrument_decorator.csv} | 0 tests/unit/test_otel_instrument.py | 118 +++--------------- tests/util/df_comparison.py | 104 +++++++++++++++ 6 files changed, 126 insertions(+), 117 deletions(-) rename tests/unit/static/golden/{test_otel_instrument__test_deterministic_app_id.csv => test_otel_instrument__test_instrument_decorator.csv} (100%) create mode 100644 tests/util/df_comparison.py diff --git a/Makefile b/Makefile index f48a785a9..84170bb67 100644 --- a/Makefile +++ b/Makefile @@ -28,15 +28,16 @@ env-%: env-tests: poetry run pip install \ - pytest \ + jsondiff \ nbconvert \ nbformat \ - pytest-subtests \ - pytest-azurepipelines \ - ruff \ + opentelemetry-sdk \ pre-commit \ + pytest \ + pytest-azurepipelines \ pytest-cov \ - jsondiff + pytest-subtests \ + ruff \ env-tests-required: poetry install --only required \ diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 82d46b627..3ded4d5b2 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -87,10 +87,6 @@ def wrapper(*args, **kwargs): try: attributes_to_add = {} - # Since we're decorating a method in a trulens app, the first argument is self, - # which we should ignore. - args = args[1:] - # Set the user provider attributes. if attributes: if callable(attributes): diff --git a/tests/test.py b/tests/test.py index 7a06a8715..19081cbd2 100644 --- a/tests/test.py +++ b/tests/test.py @@ -227,7 +227,8 @@ class WithJSONTestCase(TestCase): handling.""" def load_golden( - self, golden_path: Union[str, Path] + self, + golden_path: Union[str, Path], ) -> Union[serial_utils.JSON, pd.DataFrame]: """Load the golden file `path` and return its contents. @@ -235,7 +236,6 @@ def load_golden( golden_path: The name of the golden file to load. The file must have an extension of either `.json` or `.yaml`. The extension determines the input format. - """ golden_path = Path(golden_path) @@ -245,8 +245,6 @@ def load_golden( loader = functools.partial(yaml.load, Loader=yaml.FullLoader) elif ".csv" in golden_path.suffixes: loader = functools.partial(pd.read_csv, index_col=0) - elif ".parquet" in golden_path.suffixes: - loader = functools.partial(pd.read_parquet, index_col=0) else: raise ValueError(f"Unknown file extension {golden_path}.") diff --git a/tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv b/tests/unit/static/golden/test_otel_instrument__test_instrument_decorator.csv similarity index 100% rename from tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv rename to tests/unit/static/golden/test_otel_instrument__test_instrument_decorator.csv diff --git a/tests/unit/test_otel_instrument.py b/tests/unit/test_otel_instrument.py index 65956d655..e2678ab39 100644 --- a/tests/unit/test_otel_instrument.py +++ b/tests/unit/test_otel_instrument.py @@ -2,17 +2,20 @@ Tests for OTEL instrument decorator. """ -from typing import Any, Dict from unittest import main import pandas as pd import sqlalchemy as sa from trulens.apps.custom import TruCustomApp +from trulens.core.schema.event import EventRecordType from trulens.core.session import TruSession from trulens.experimental.otel_tracing.core.init import init from trulens.experimental.otel_tracing.core.instrument import instrument from tests.test import TruTestCase +from tests.util.df_comparison import ( + compare_dfs_accounting_for_ids_and_timestamps, +) class _TestApp: @@ -55,9 +58,6 @@ def nested3(self, query: str) -> str: class TestOtelInstrument(TruTestCase): - def setUp(self): - pass - @staticmethod def _get_events() -> pd.DataFrame: tru_session = TruSession() @@ -68,8 +68,14 @@ def _get_events() -> pd.DataFrame: @staticmethod def _convert_column_types(df: pd.DataFrame): + # Writing to CSV and the reading back causes some type issues so we + # hackily convert things here. df["event_id"] = df["event_id"].apply(str) - df["record_type"] = df["record_type"].apply(lambda x: eval(x)) + df["record_type"] = df["record_type"].apply( + lambda x: EventRecordType(x[len("EventRecordType.") :]) + if x.startswith("EventRecordType.") + else EventRecordType(x) + ) df["start_timestamp"] = df["start_timestamp"].apply(pd.Timestamp) df["timestamp"] = df["timestamp"].apply(pd.Timestamp) for json_column in [ @@ -80,103 +86,7 @@ def _convert_column_types(df: pd.DataFrame): ]: df[json_column] = df[json_column].apply(lambda x: eval(x)) - def _compare_dfs_accounting_for_ids_and_timestamps( - self, expected: pd.DataFrame, actual: pd.DataFrame - ): - """ - Compare two Dataframes are equal, accounting for ids and timestamps. - That is: - 1. The ids between the two Dataframes may be different, but they have - to be consistent. That is, if one Dataframe reuses an id in two - places, then the other must as well. - 2. The timestamps between the two Dataframes may be different, but - they have to be in the same order. - - Args: - expected: expected results - actual: actual results - """ - id_mapping: Dict[str, str] = {} - timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp] = {} - self.assertEqual(len(expected), len(actual)) - self.assertListEqual(list(expected.columns), list(actual.columns)) - for i in range(len(expected)): - for col in expected.columns: - self._compare_entity( - expected.iloc[i][col], - actual.iloc[i][col], - id_mapping, - timestamp_mapping, - is_id=col.endswith("_id"), - locator=f"df.iloc[{i}][{col}]", - ) - # Ensure that the id mapping is a bijection. - self.assertEqual( - len(set(id_mapping.values())), - len(id_mapping), - "Ids are not a bijection!", - ) - # Ensure that the timestamp mapping is monotonic. - prev_value = None - for curr in sorted(timestamp_mapping.keys()): - if prev_value is not None: - self.assertLess( - prev_value, - timestamp_mapping[curr], - "Timestamps are not in the same order!", - ) - prev_value = timestamp_mapping[curr] - - def _compare_entity( - self, - expected: Any, - actual: Any, - id_mapping: Dict[str, str], - timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp], - is_id: bool, - locator: str, - ): - self.assertEqual( - type(expected), type(actual), f"Types of {locator} do not match!" - ) - if is_id: - self.assertEqual( - type(expected), str, f"Type of id {locator} is not a string!" - ) - if expected not in id_mapping: - id_mapping[expected] = actual - self.assertEqual( - id_mapping[expected], - actual, - f"Ids of {locator} are not consistent!", - ) - elif isinstance(expected, dict): - self.assertEqual( - expected.keys(), - actual.keys(), - f"Keys of {locator} do not match!", - ) - for k in expected.keys(): - self._compare_entity( - expected[k], - actual[k], - id_mapping, - timestamp_mapping, - is_id=k.endswith("_id"), - locator=f"{locator}[k]", - ) - elif isinstance(expected, pd.Timestamp): - if expected not in timestamp_mapping: - timestamp_mapping[expected] = actual - self.assertEqual( - timestamp_mapping[expected], - actual, - f"Timestamps of {locator} are not consistent!", - ) - else: - self.assertEqual(expected, actual, f"{locator} does not match!") - - def test_deterministic_app_id(self): + def test_instrument_decorator(self): # Set up. tru_session = TruSession() tru_session.experimental_enable_feature("otel_tracing") @@ -190,13 +100,13 @@ def test_deterministic_app_id(self): with custom_app: test_app.respond_to_query("throw") # Compare results to expected. - GOLDEN_FILENAME = "tests/unit/static/golden/test_otel_instrument__test_deterministic_app_id.csv" + GOLDEN_FILENAME = "tests/unit/static/golden/test_otel_instrument__test_instrument_decorator.csv" actual = self._get_events() self.assertEqual(len(actual), 8) self.write_golden(GOLDEN_FILENAME, actual) expected = self.load_golden(GOLDEN_FILENAME) self._convert_column_types(expected) - self._compare_dfs_accounting_for_ids_and_timestamps(expected, actual) + compare_dfs_accounting_for_ids_and_timestamps(self, expected, actual) if __name__ == "__main__": diff --git a/tests/util/df_comparison.py b/tests/util/df_comparison.py new file mode 100644 index 000000000..381658ca5 --- /dev/null +++ b/tests/util/df_comparison.py @@ -0,0 +1,104 @@ +from typing import Any, Dict +from unittest import TestCase + +import pandas as pd + + +def compare_dfs_accounting_for_ids_and_timestamps( + test_case: TestCase, expected: pd.DataFrame, actual: pd.DataFrame +): + """ + Compare two Dataframes are equal, accounting for ids and timestamps. That + is: + 1. The ids between the two Dataframes may be different, but they have to be + consistent. That is, if one Dataframe reuses an id in two places, then + the other must as well. + 2. The timestamps between the two Dataframes may be different, but they + have to be in the same order. + + Args: + expected: expected results + actual: actual results + """ + id_mapping: Dict[str, str] = {} + timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp] = {} + test_case.assertEqual(len(expected), len(actual)) + test_case.assertListEqual(list(expected.columns), list(actual.columns)) + for i in range(len(expected)): + for col in expected.columns: + _compare_entity( + test_case, + expected.iloc[i][col], + actual.iloc[i][col], + id_mapping, + timestamp_mapping, + is_id=col.endswith("_id"), + locator=f"df.iloc[{i}][{col}]", + ) + # Ensure that the id mapping is a bijection. + test_case.assertEqual( + len(set(id_mapping.values())), + len(id_mapping), + "Ids are not a bijection!", + ) + # Ensure that the timestamp mapping is strictly increasing. + prev_value = None + for curr in sorted(timestamp_mapping.keys()): + if prev_value is not None: + test_case.assertLess( + prev_value, + timestamp_mapping[curr], + "Timestamps are not in the same order!", + ) + prev_value = timestamp_mapping[curr] + + +def _compare_entity( + test_case: TestCase, + expected: Any, + actual: Any, + id_mapping: Dict[str, str], + timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp], + is_id: bool, + locator: str, +): + test_case.assertEqual( + type(expected), type(actual), f"Types of {locator} do not match!" + ) + if is_id: + test_case.assertEqual( + type(expected), str, f"Type of id {locator} is not a string!" + ) + if expected not in id_mapping: + id_mapping[expected] = actual + test_case.assertEqual( + id_mapping[expected], + actual, + f"Ids of {locator} are not consistent!", + ) + elif isinstance(expected, dict): + test_case.assertEqual( + expected.keys(), + actual.keys(), + f"Keys of {locator} do not match!", + ) + for k in expected.keys(): + _compare_entity( + test_case, + expected[k], + actual[k], + id_mapping, + timestamp_mapping, + is_id=k.endswith("_id"), + locator=f"{locator}[k]", + ) + elif isinstance(expected, pd.Timestamp): + if expected not in timestamp_mapping: + timestamp_mapping[expected] = actual + test_case.assertEqual( + timestamp_mapping[expected], + actual, + f"Timestamps of {locator} are not consistent!", + ) + else: + test_case.assertEqual(expected, actual, f"{locator} does not match!") From 78148076570407f96594b1c075b5cbab3de7796a Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sun, 22 Dec 2024 11:01:02 -0800 Subject: [PATCH 42/59] Use `Dict` instead of `dict` for types. --- .../experimental/otel_tracing/core/instrument.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 3ded4d5b2..b1a2682ef 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -1,6 +1,6 @@ from functools import wraps import logging -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Dict, Optional, Union from opentelemetry import trace from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME @@ -13,9 +13,9 @@ def instrument( *, attributes: Optional[ Union[ - dict[str, Any], + Dict[str, Any], Callable[ - [Optional[Any], Optional[Exception], Any, Any], dict[str, Any] + [Optional[Any], Optional[Exception], Any, Any], Dict[str, Any] ], ] ] = {}, @@ -25,7 +25,7 @@ def instrument( wrapped by TruCustomApp, with OpenTelemetry tracing. """ - def _validate_selector_name(attributes: dict[str, Any]) -> dict[str, Any]: + def _validate_selector_name(attributes: Dict[str, Any]) -> Dict[str, Any]: result = attributes.copy() if ( @@ -52,7 +52,7 @@ def _validate_selector_name(attributes: dict[str, Any]) -> dict[str, Any]: return result - def _validate_attributes(attributes: dict[str, Any]) -> dict[str, Any]: + def _validate_attributes(attributes: Dict[str, Any]) -> Dict[str, Any]: if not isinstance(attributes, dict) or any([ not isinstance(key, str) for key in attributes.keys() ]): From c390f94a2620022d1e2a175b40310a737d28ac31 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sun, 22 Dec 2024 11:21:22 -0800 Subject: [PATCH 43/59] Allow otel test to run with experimental flag. --- ..._instrument__test_instrument_decorator.csv | 16 +++++----- tests/unit/test_otel_instrument.py | 30 ++++++++++++++++--- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/tests/unit/static/golden/test_otel_instrument__test_instrument_decorator.csv b/tests/unit/static/golden/test_otel_instrument__test_instrument_decorator.csv index f03fabb32..40837d509 100644 --- a/tests/unit/static/golden/test_otel_instrument__test_instrument_decorator.csv +++ b/tests/unit/static/golden/test_otel_instrument__test_instrument_decorator.csv @@ -1,9 +1,9 @@ ,record,event_id,record_attributes,record_type,resource_attributes,start_timestamp,timestamp,trace -0,"{'name': 'respond_to_query', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '', 'status': 'STATUS_CODE_UNSET'}",11560964145073308497,{},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817734,2024-12-21 22:46:48.822087,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '', 'span_id': '11560964145073308497'}" -1,"{'name': 'nested', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '11560964145073308497', 'status': 'STATUS_CODE_UNSET'}",956390419060041970,{'trulens.nested_attr1': 'value1'},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817764,2024-12-21 22:46:48.821018,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '11560964145073308497', 'span_id': '956390419060041970'}" -2,"{'name': 'nested2', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '956390419060041970', 'status': 'STATUS_CODE_UNSET'}",7191373700275539380,"{'trulens.nested2_ret': 'nested2: nested3', 'trulens.nested2_args[0]': 'test'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817784,2024-12-21 22:46:48.819862,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '956390419060041970', 'span_id': '7191373700275539380'}" -3,"{'name': 'nested3', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '7191373700275539380', 'status': 'STATUS_CODE_UNSET'}",11561410003717533750,"{'trulens.nested3_ret': 'nested3', 'trulens.special.nested3_ret': 'nested3', 'trulens.selector_name': 'special', 'trulens.cows': 'moo', 'trulens.special.cows': 'moo'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.817801,2024-12-21 22:46:48.817858,"{'trace_id': '105357914249406038608753130269542497721', 'parent_id': '7191373700275539380', 'span_id': '11561410003717533750'}" -4,"{'name': 'respond_to_query', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '', 'status': 'STATUS_CODE_UNSET'}",15712161957924150536,{},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823641,2024-12-21 22:46:48.827118,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '', 'span_id': '15712161957924150536'}" -5,"{'name': 'nested', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '15712161957924150536', 'status': 'STATUS_CODE_UNSET'}",14875482935475817656,{'trulens.nested_attr1': 'value1'},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823666,2024-12-21 22:46:48.826205,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '15712161957924150536', 'span_id': '14875482935475817656'}" -6,"{'name': 'nested2', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '14875482935475817656', 'status': 'STATUS_CODE_UNSET'}",7500502012444675242,"{'trulens.nested2_ret': 'nested2: ', 'trulens.nested2_args[0]': 'throw'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823683,2024-12-21 22:46:48.825310,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '14875482935475817656', 'span_id': '7500502012444675242'}" -7,"{'name': 'nested3', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '7500502012444675242', 'status': 'STATUS_CODE_ERROR'}",9125451447180209633,"{'trulens.nested3_ex': ['nested3 exception'], 'trulens.special.nested3_ex': ['nested3 exception'], 'trulens.selector_name': 'special', 'trulens.cows': 'moo', 'trulens.special.cows': 'moo'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-21 22:46:48.823710,2024-12-21 22:46:48.824262,"{'trace_id': '311544861227161086009958871975634544174', 'parent_id': '7500502012444675242', 'span_id': '9125451447180209633'}" +0,"{'name': 'respond_to_query', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '', 'status': 'STATUS_CODE_UNSET'}",7870250274962447839,{},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.387607,2024-12-22 11:20:26.392174,"{'trace_id': '113376089399064103615948241236196474059', 'parent_id': '', 'span_id': '7870250274962447839'}" +1,"{'name': 'nested', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '7870250274962447839', 'status': 'STATUS_CODE_UNSET'}",8819384298151247754,{'trulens.nested_attr1': 'value1'},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.387652,2024-12-22 11:20:26.391272,"{'trace_id': '113376089399064103615948241236196474059', 'parent_id': '7870250274962447839', 'span_id': '8819384298151247754'}" +2,"{'name': 'nested2', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '8819384298151247754', 'status': 'STATUS_CODE_UNSET'}",2622992513876904334,"{'trulens.nested2_ret': 'nested2: nested3', 'trulens.nested2_args[1]': 'test'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.387679,2024-12-22 11:20:26.389939,"{'trace_id': '113376089399064103615948241236196474059', 'parent_id': '8819384298151247754', 'span_id': '2622992513876904334'}" +3,"{'name': 'nested3', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '2622992513876904334', 'status': 'STATUS_CODE_UNSET'}",11864485227397090485,"{'trulens.nested3_ret': 'nested3', 'trulens.special.nested3_ret': 'nested3', 'trulens.selector_name': 'special', 'trulens.cows': 'moo', 'trulens.special.cows': 'moo'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.387705,2024-12-22 11:20:26.387762,"{'trace_id': '113376089399064103615948241236196474059', 'parent_id': '2622992513876904334', 'span_id': '11864485227397090485'}" +4,"{'name': 'respond_to_query', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '', 'status': 'STATUS_CODE_UNSET'}",10786111609955477438,{},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.393563,2024-12-22 11:20:26.397446,"{'trace_id': '214293944471171141309178747794638512671', 'parent_id': '', 'span_id': '10786111609955477438'}" +5,"{'name': 'nested', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '10786111609955477438', 'status': 'STATUS_CODE_UNSET'}",7881765616183808794,{'trulens.nested_attr1': 'value1'},EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.393586,2024-12-22 11:20:26.396613,"{'trace_id': '214293944471171141309178747794638512671', 'parent_id': '10786111609955477438', 'span_id': '7881765616183808794'}" +6,"{'name': 'nested2', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '7881765616183808794', 'status': 'STATUS_CODE_UNSET'}",4318803655649897130,"{'trulens.nested2_ret': 'nested2: ', 'trulens.nested2_args[1]': 'throw'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.393603,2024-12-22 11:20:26.395227,"{'trace_id': '214293944471171141309178747794638512671', 'parent_id': '7881765616183808794', 'span_id': '4318803655649897130'}" +7,"{'name': 'nested3', 'kind': 'SPAN_KIND_TRULENS', 'parent_span_id': '4318803655649897130', 'status': 'STATUS_CODE_ERROR'}",11457830288984624191,"{'trulens.nested3_ex': ['nested3 exception'], 'trulens.special.nested3_ex': ['nested3 exception'], 'trulens.selector_name': 'special', 'trulens.cows': 'moo', 'trulens.special.cows': 'moo'}",EventRecordType.SPAN,"{'telemetry.sdk.language': 'python', 'telemetry.sdk.name': 'opentelemetry', 'telemetry.sdk.version': '1.28.2', 'service.name': 'trulens'}",2024-12-22 11:20:26.393630,2024-12-22 11:20:26.394348,"{'trace_id': '214293944471171141309178747794638512671', 'parent_id': '4318803655649897130', 'span_id': '11457830288984624191'}" diff --git a/tests/unit/test_otel_instrument.py b/tests/unit/test_otel_instrument.py index e2678ab39..e01392fc1 100644 --- a/tests/unit/test_otel_instrument.py +++ b/tests/unit/test_otel_instrument.py @@ -30,7 +30,7 @@ def nested(self, query: str) -> str: @instrument( attributes=lambda ret, exception, *args, **kwargs: { "nested2_ret": ret, - "nested2_args[0]": args[0], + "nested2_args[1]": args[1], } ) def nested2(self, query: str) -> str: @@ -58,6 +58,29 @@ def nested3(self, query: str) -> str: class TestOtelInstrument(TruTestCase): + @classmethod + def clear_TruSession_singleton(cls) -> None: + # [HACK!] Clean up any instances of `TruSession` so tests don't + # interfere with each other. + for key in [ + curr + for curr in TruSession._singleton_instances + if curr[0] == "trulens.core.session.TruSession" + ]: + del TruSession._singleton_instances[key] + + @classmethod + def setUpClass(cls) -> None: + cls.clear_TruSession_singleton() + tru_session = TruSession() + tru_session.experimental_enable_feature("otel_tracing") + return super().setUpClass() + + @classmethod + def tearDownClass(cls) -> None: + cls.clear_TruSession_singleton() + return super().tearDownClass() + @staticmethod def _get_events() -> pd.DataFrame: tru_session = TruSession() @@ -67,7 +90,7 @@ def _get_events() -> pd.DataFrame: return pd.read_sql(q, db_session.bind) @staticmethod - def _convert_column_types(df: pd.DataFrame): + def _convert_column_types(df: pd.DataFrame) -> None: # Writing to CSV and the reading back causes some type issues so we # hackily convert things here. df["event_id"] = df["event_id"].apply(str) @@ -86,10 +109,9 @@ def _convert_column_types(df: pd.DataFrame): ]: df[json_column] = df[json_column].apply(lambda x: eval(x)) - def test_instrument_decorator(self): + def test_instrument_decorator(self) -> None: # Set up. tru_session = TruSession() - tru_session.experimental_enable_feature("otel_tracing") tru_session.reset_database() init(tru_session, debug=True) # Create and run app. From fe12d1bc74826943794de6935111d4840f75bb68 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sun, 22 Dec 2024 11:30:08 -0800 Subject: [PATCH 44/59] Don't compare sdk versions. --- tests/unit/test_otel_instrument.py | 9 ++++++++- tests/util/df_comparison.py | 20 +++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_otel_instrument.py b/tests/unit/test_otel_instrument.py index e01392fc1..55d8a7bc5 100644 --- a/tests/unit/test_otel_instrument.py +++ b/tests/unit/test_otel_instrument.py @@ -128,7 +128,14 @@ def test_instrument_decorator(self) -> None: self.write_golden(GOLDEN_FILENAME, actual) expected = self.load_golden(GOLDEN_FILENAME) self._convert_column_types(expected) - compare_dfs_accounting_for_ids_and_timestamps(self, expected, actual) + compare_dfs_accounting_for_ids_and_timestamps( + self, + expected, + actual, + ignore_locators=[ + "df.iloc[0][resource_attributes][telemetry.sdk.version]" + ], + ) if __name__ == "__main__": diff --git a/tests/util/df_comparison.py b/tests/util/df_comparison.py index 381658ca5..757f05df6 100644 --- a/tests/util/df_comparison.py +++ b/tests/util/df_comparison.py @@ -1,12 +1,15 @@ -from typing import Any, Dict +from typing import Any, Dict, Optional, Sequence from unittest import TestCase import pandas as pd def compare_dfs_accounting_for_ids_and_timestamps( - test_case: TestCase, expected: pd.DataFrame, actual: pd.DataFrame -): + test_case: TestCase, + expected: pd.DataFrame, + actual: pd.DataFrame, + ignore_locators: Optional[Sequence[str]], +) -> None: """ Compare two Dataframes are equal, accounting for ids and timestamps. That is: @@ -17,8 +20,10 @@ def compare_dfs_accounting_for_ids_and_timestamps( have to be in the same order. Args: + test_case: unittest.TestCase instance to use for assertions expected: expected results actual: actual results + ignore_locators: locators to ignore when comparing the Dataframes """ id_mapping: Dict[str, str] = {} timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp] = {} @@ -34,6 +39,7 @@ def compare_dfs_accounting_for_ids_and_timestamps( timestamp_mapping, is_id=col.endswith("_id"), locator=f"df.iloc[{i}][{col}]", + ignore_locators=ignore_locators, ) # Ensure that the id mapping is a bijection. test_case.assertEqual( @@ -61,7 +67,10 @@ def _compare_entity( timestamp_mapping: Dict[pd.Timestamp, pd.Timestamp], is_id: bool, locator: str, -): + ignore_locators: Optional[Sequence[str]], +) -> None: + if locator in ignore_locators: + return test_case.assertEqual( type(expected), type(actual), f"Types of {locator} do not match!" ) @@ -90,7 +99,8 @@ def _compare_entity( id_mapping, timestamp_mapping, is_id=k.endswith("_id"), - locator=f"{locator}[k]", + locator=f"{locator}[{k}]", + ignore_locators=ignore_locators, ) elif isinstance(expected, pd.Timestamp): if expected not in timestamp_mapping: From a7e3f53b6ae2c96f3fe3cbe07e7d7dc383c59b72 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sun, 22 Dec 2024 20:37:43 -0800 Subject: [PATCH 45/59] Handle all rows that need to have the sdk version ignored. --- tests/unit/test_otel_instrument.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_otel_instrument.py b/tests/unit/test_otel_instrument.py index 55d8a7bc5..cc6b37b21 100644 --- a/tests/unit/test_otel_instrument.py +++ b/tests/unit/test_otel_instrument.py @@ -133,7 +133,8 @@ def test_instrument_decorator(self) -> None: expected, actual, ignore_locators=[ - "df.iloc[0][resource_attributes][telemetry.sdk.version]" + f"df.iloc[{i}][resource_attributes][telemetry.sdk.version]" + for i in range(8) ], ) From 5dbc83ad1f7ff34e0ebd50aba81428cd2ee8f9e6 Mon Sep 17 00:00:00 2001 From: David Kurokawa Date: Sun, 22 Dec 2024 20:43:30 -0800 Subject: [PATCH 46/59] Fix handling `None` issue for `ignore_locators` arg. --- tests/util/df_comparison.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/util/df_comparison.py b/tests/util/df_comparison.py index 757f05df6..321edf586 100644 --- a/tests/util/df_comparison.py +++ b/tests/util/df_comparison.py @@ -69,7 +69,7 @@ def _compare_entity( locator: str, ignore_locators: Optional[Sequence[str]], ) -> None: - if locator in ignore_locators: + if ignore_locators and locator in ignore_locators: return test_case.assertEqual( type(expected), type(actual), f"Types of {locator} do not match!" From 3f1607cbcadd83cb2b93ed60335761afe00ce6c2 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Mon, 16 Dec 2024 14:03:32 -0500 Subject: [PATCH 47/59] save --- examples/experimental/otel_exporter.ipynb | 302 +++++++++--------- src/core/trulens/core/app.py | 13 +- .../otel_tracing/core/instrument.py | 18 ++ 3 files changed, 177 insertions(+), 156 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index a3a145b3e..18859e594 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -1,154 +1,154 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install opentelemetry-api\n", - "# !pip install opentelemetry-sdk" - ] + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install opentelemetry-api\n", + "# !pip install opentelemetry-sdk" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Add base dir to path to be able to access test folder.\n", + "base_dir = Path().cwd().parent.parent.resolve()\n", + "if str(base_dir) not in sys.path:\n", + " print(f\"Adding {base_dir} to sys.path\")\n", + " sys.path.append(str(base_dir))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "root = logging.getLogger()\n", + "root.setLevel(logging.DEBUG)\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setLevel(logging.DEBUG)\n", + "handler.addFilter(logging.Filter(\"trulens\"))\n", + "formatter = logging.Formatter(\n", + " \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n", + ")\n", + "handler.setFormatter(formatter)\n", + "root.addHandler(handler)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.experimental.otel_tracing.core.instrument import instrument\n", + "\n", + "\n", + "class TestApp:\n", + " @instrument()\n", + " def respond_to_query(self, query: str) -> str:\n", + " return f\"answer: {self.nested(query)}\"\n", + "\n", + " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", + " def nested(self, query: str) -> str:\n", + " return f\"nested: {self.nested2(query)}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, exception, *args, **kwargs: {\n", + " \"nested2_ret\": ret,\n", + " \"nested2_args[0]\": args[0],\n", + " }\n", + " )\n", + " def nested2(self, query: str) -> str:\n", + " nested_result = \"\"\n", + "\n", + " try:\n", + " nested_result = self.nested3(query)\n", + " except Exception:\n", + " pass\n", + "\n", + " return f\"nested2: {nested_result}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, exception, *args, **kwargs: {\n", + " \"nested3_ex\": exception.args if exception else None,\n", + " \"nested3_ret\": ret,\n", + " \"selector_name\": \"special\",\n", + " \"cows\": \"moo\",\n", + " }\n", + " )\n", + " def nested3(self, query: str) -> str:\n", + " if query == \"throw\":\n", + " raise ValueError(\"nested3 exception\")\n", + " return \"nested3\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import dotenv\n", + "from trulens.core.session import TruSession\n", + "from trulens.experimental.otel_tracing.core.init import init\n", + "\n", + "dotenv.load_dotenv()\n", + "\n", + "session = TruSession()\n", + "session.experimental_enable_feature(\"otel_tracing\")\n", + "session.reset_database()\n", + "init(session, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.apps.custom import TruCustomApp\n", + "\n", + "test_app = TestApp()\n", + "custom_app = TruCustomApp(test_app)\n", + "\n", + "with custom_app as recording:\n", + " test_app.respond_to_query(\"test\")\n", + "\n", + "with custom_app as recording:\n", + " test_app.respond_to_query(\"throw\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "trulens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import sys\n", - "\n", - "# Add base dir to path to be able to access test folder.\n", - "base_dir = Path().cwd().parent.parent.resolve()\n", - "if str(base_dir) not in sys.path:\n", - " print(f\"Adding {base_dir} to sys.path\")\n", - " sys.path.append(str(base_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "\n", - "root = logging.getLogger()\n", - "root.setLevel(logging.DEBUG)\n", - "handler = logging.StreamHandler(sys.stdout)\n", - "handler.setLevel(logging.DEBUG)\n", - "handler.addFilter(logging.Filter(\"trulens\"))\n", - "formatter = logging.Formatter(\n", - " \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n", - ")\n", - "handler.setFormatter(formatter)\n", - "root.addHandler(handler)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.experimental.otel_tracing.core.instrument import instrument\n", - "\n", - "\n", - "class TestApp:\n", - " @instrument()\n", - " def respond_to_query(self, query: str) -> str:\n", - " return f\"answer: {self.nested(query)}\"\n", - "\n", - " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", - " def nested(self, query: str) -> str:\n", - " return f\"nested: {self.nested2(query)}\"\n", - "\n", - " @instrument(\n", - " attributes=lambda ret, exception, *args, **kwargs: {\n", - " \"nested2_ret\": ret,\n", - " \"nested2_args[0]\": args[0],\n", - " }\n", - " )\n", - " def nested2(self, query: str) -> str:\n", - " nested_result = \"\"\n", - "\n", - " try:\n", - " nested_result = self.nested3(query)\n", - " except Exception:\n", - " pass\n", - "\n", - " return f\"nested2: {nested_result}\"\n", - "\n", - " @instrument(\n", - " attributes=lambda ret, exception, *args, **kwargs: {\n", - " \"nested3_ex\": exception.args if exception else None,\n", - " \"nested3_ret\": ret,\n", - " \"selector_name\": \"special\",\n", - " \"cows\": \"moo\",\n", - " }\n", - " )\n", - " def nested3(self, query: str) -> str:\n", - " if query == \"throw\":\n", - " raise ValueError(\"nested3 exception\")\n", - " return \"nested3\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import dotenv\n", - "from trulens.core.session import TruSession\n", - "from trulens.experimental.otel_tracing.core.init import init\n", - "\n", - "dotenv.load_dotenv()\n", - "\n", - "session = TruSession()\n", - "session.experimental_enable_feature(\"otel_tracing\")\n", - "session.reset_database()\n", - "init(session, debug=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.apps.custom import TruCustomApp\n", - "\n", - "test_app = TestApp()\n", - "custom_app = TruCustomApp(test_app)\n", - "\n", - "with custom_app as recording:\n", - " test_app.respond_to_query(\"test\")\n", - "\n", - "with custom_app as recording:\n", - " test_app.respond_to_query(\"throw\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "trulens", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/src/core/trulens/core/app.py b/src/core/trulens/core/app.py index f27ac2cdd..cfeda51e5 100644 --- a/src/core/trulens/core/app.py +++ b/src/core/trulens/core/app.py @@ -1048,15 +1048,16 @@ def __enter__(self): if self.session.experimental_feature( core_experimental.Feature.OTEL_TRACING ): - from trulens.experimental.otel_tracing.core.app import _App + from trulens.experimental.otel_tracing.core.instrument import ( + App as OTELApp, + ) - return _App.__enter__(self) + return OTELApp.__enter__(self) ctx = core_instruments._RecordingContext(app=self) token = self.recording_contexts.set(ctx) ctx.token = token - # self._set_context_vars() return ctx @@ -1066,9 +1067,11 @@ def __exit__(self, exc_type, exc_value, exc_tb): if self.session.experimental_feature( core_experimental.Feature.OTEL_TRACING ): - from trulens.experimental.otel_tracing.core.app import _App + from trulens.experimental.otel_tracing.core.instrument import ( + App as OTELApp, + ) - return _App.__exit__(self, exc_type, exc_value, exc_tb) + return OTELApp.__exit__(self, exc_type, exc_value, exc_tb) ctx = self.recording_contexts.get() self.recording_contexts.reset(ctx.token) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index b1a2682ef..8f540bde6 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -3,6 +3,8 @@ from typing import Any, Callable, Dict, Optional, Union from opentelemetry import trace +from trulens.apps.custom import instrument as custom_instrument +from trulens.core import app as core_app from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME from trulens.otel.semconv.trace import SpanAttributes @@ -137,3 +139,19 @@ def wrapper(*args, **kwargs): return wrapper return inner_decorator + + +class App(core_app.App): + # For use as a context manager. + def __enter__(self): + return ( + trace.get_tracer_provider() + .get_tracer(TRULENS_SERVICE_NAME) + .start_as_current_span( + name="root", + ) + .__enter__() + ) + + def __exit__(self, exc_type, exc_value, exc_tb): + print("exit") From 2a6e699176e334d93bd5c352b54aa44da101c265 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 19 Dec 2024 12:25:03 -0500 Subject: [PATCH 48/59] draft --- examples/experimental/otel_exporter.ipynb | 136 +++++++++++++++++- src/core/trulens/core/app.py | 25 +++- .../otel_tracing/core/instrument.py | 87 +++++++++-- .../semconv/trulens/otel/semconv/trace.py | 7 + 4 files changed, 240 insertions(+), 15 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 18859e594..67b972b42 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -149,6 +149,138 @@ "pygments_lexer": "ipython3" } }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Add base dir to path to be able to access test folder.\n", + "base_dir = Path().cwd().parent.parent.resolve()\n", + "if str(base_dir) not in sys.path:\n", + " print(f\"Adding {base_dir} to sys.path\")\n", + " sys.path.append(str(base_dir))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "root = logging.getLogger()\n", + "root.setLevel(logging.DEBUG)\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setLevel(logging.DEBUG)\n", + "handler.addFilter(logging.Filter(\"trulens\"))\n", + "formatter = logging.Formatter(\n", + " \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n", + ")\n", + "handler.setFormatter(formatter)\n", + "root.addHandler(handler)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.experimental.otel_tracing.core.instrument import instrument\n", + "\n", + "\n", + "class TestApp:\n", + " @instrument()\n", + " def respond_to_query(self, query: str) -> str:\n", + " return f\"answer: {self.nested(query)}\"\n", + "\n", + " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", + " def nested(self, query: str) -> str:\n", + " return f\"nested: {self.nested2(query)}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, *args, **kwargs: {\n", + " \"nested2_ret\": ret,\n", + " \"nested2_args[0]\": args[0],\n", + " }\n", + " )\n", + " def nested2(self, query: str) -> str:\n", + " return f\"nested2: {query}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import dotenv\n", + "from trulens.connectors.snowflake import SnowflakeConnector\n", + "from trulens.core.session import TruSession\n", + "from trulens.experimental.otel_tracing.core.init import init\n", + "\n", + "dotenv.load_dotenv()\n", + "\n", + "connection_params = {\n", + " \"account\": os.environ[\"SNOWFLAKE_ACCOUNT\"],\n", + " \"user\": os.environ[\"SNOWFLAKE_USER\"],\n", + " \"password\": os.environ[\"SNOWFLAKE_USER_PASSWORD\"],\n", + " \"database\": os.environ[\"SNOWFLAKE_DATABASE\"],\n", + " \"schema\": os.environ[\"SNOWFLAKE_SCHEMA\"],\n", + " \"warehouse\": os.environ[\"SNOWFLAKE_WAREHOUSE\"],\n", + " \"role\": os.environ[\"SNOWFLAKE_ROLE\"],\n", + "}\n", + "\n", + "connector = SnowflakeConnector(\n", + " **connection_params, database_redact_keys=True, database_args=None\n", + ")\n", + "session = TruSession(connector=connector)\n", + "session.experimental_enable_feature(\"otel_tracing\")\n", + "session.reset_database()\n", + "init(session, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trulens.apps.custom import TruCustomApp\n", + "\n", + "test_app = TestApp()\n", + "custom_app = TruCustomApp(test_app)\n", + "\n", + "with custom_app as recording:\n", + " test_app.respond_to_query(\"test\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "trulens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/src/core/trulens/core/app.py b/src/core/trulens/core/app.py index cfeda51e5..7fc316a8c 100644 --- a/src/core/trulens/core/app.py +++ b/src/core/trulens/core/app.py @@ -3,6 +3,7 @@ from abc import ABC from abc import ABCMeta from abc import abstractmethod +import contextlib import contextvars import datetime import inspect @@ -421,6 +422,17 @@ def tru(self) -> core_connector.DBConnector: pydantic.PrivateAttr(default_factory=dict) ) + token: Optional[object] = None + """ + OTEL context token for the current context manager. + """ + + span_context: Optional[contextlib.AbstractContextManager] = None + """ + Span context manager. Required to help keep track of the appropriate span context + to enter/exit. + """ + def __init__( self, connector: Optional[core_connector.DBConnector] = None, @@ -1058,7 +1070,6 @@ def __enter__(self): token = self.recording_contexts.set(ctx) ctx.token = token - # self._set_context_vars() return ctx @@ -1088,9 +1099,11 @@ async def __aenter__(self): if self.session.experimental_feature( core_experimental.Feature.OTEL_TRACING ): - from trulens.experimental.otel_tracing.core.app import _App + from trulens.experimental.otel_tracing.core.instrument import ( + App as OTELApp, + ) - return await _App.__aenter__(self) + return OTELApp.__enter__(self) ctx = core_instruments._RecordingContext(app=self) @@ -1106,9 +1119,11 @@ async def __aexit__(self, exc_type, exc_value, exc_tb): if self.session.experimental_feature( core_experimental.Feature.OTEL_TRACING ): - from trulens.experimental.otel_tracing.core.app import _App + from trulens.experimental.otel_tracing.core.instrument import ( + App as OTELApp, + ) - return await _App.__aexit__(self, exc_type, exc_value, exc_tb) + return OTELApp.__exit__(self, exc_type, exc_value, exc_tb) ctx = self.recording_contexts.get() self.recording_contexts.reset(ctx.token) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 8f540bde6..56947f14a 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -1,11 +1,26 @@ from functools import wraps import logging +<<<<<<< HEAD from typing import Any, Callable, Dict, Optional, Union +======= +from typing import Any, Callable, Optional, Union +import uuid +>>>>>>> ae0e4d895 (draft) from opentelemetry import trace +from opentelemetry.baggage import get_baggage +from opentelemetry.baggage import remove_baggage +from opentelemetry.baggage import set_baggage +import opentelemetry.context as context_api from trulens.apps.custom import instrument as custom_instrument from trulens.core import app as core_app from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME +<<<<<<< HEAD +======= +from trulens.experimental.otel_tracing.core.semantic import ( + TRULENS_SELECTOR_NAME, +) +>>>>>>> 41ef4d524 (draft) from trulens.otel.semconv.trace import SpanAttributes logger = logging.getLogger(__name__) @@ -79,12 +94,30 @@ def wrapper(*args, **kwargs): func_exception: Optional[Exception] = None attributes_exception: Optional[Exception] = None +<<<<<<< HEAD try: ret = func(*args, **kwargs) except Exception as e: # We want to get into the next clause to allow the users to still add attributes. # It's on the user to deal with None as a return value. +<<<<<<< HEAD func_exception = e +======= + exception = e +======= + span.set_attribute("name", func.__name__) + span.set_attribute("kind", "SPAN_KIND_TRULENS") + span.set_attribute( + "parent_span_id", parent_span.get_span_context().span_id + ) + span.set_attribute( + SpanAttributes.RECORD_ID, + str(get_baggage(SpanAttributes.RECORD_ID)), + ) + + ret = func(*args, **kwargs) +>>>>>>> 41ef4d524 (draft) +>>>>>>> ae0e4d895 (draft) try: attributes_to_add = {} @@ -144,14 +177,52 @@ def wrapper(*args, **kwargs): class App(core_app.App): # For use as a context manager. def __enter__(self): - return ( - trace.get_tracer_provider() - .get_tracer(TRULENS_SERVICE_NAME) - .start_as_current_span( - name="root", - ) - .__enter__() + logging.debug("Entering the OTEL app context.") + + # Note: This is not the same as the record_id in the core app since the OTEL + # tracing is currently separate from the old records behavior + otel_record_id = str(uuid.uuid4()) + + tracer = trace.get_tracer_provider().get_tracer(TRULENS_SERVICE_NAME) + + # Calling set_baggage does not actually add the baggage to the current context, but returns a new one + # To avoid issues with remembering to add/remove the baggage, we attach it to the runtime context. + self.token = context_api.attach( + set_baggage(SpanAttributes.RECORD_ID, otel_record_id) ) + # Use start_as_current_span as a context manager + self.span_context = tracer.start_as_current_span("root") + root_span = self.span_context.__enter__() + + logger.debug(str(get_baggage(SpanAttributes.RECORD_ID))) + + root_span.set_attribute("kind", "SPAN_KIND_TRULENS") + root_span.set_attribute("name", "root") + root_span.set_attribute( + SpanAttributes.SPAN_TYPE, SpanAttributes.SpanType.RECORD_ROOT + ) + root_span.set_attribute( + SpanAttributes.RECORD_ROOT.APP_NAME, self.app_name + ) + root_span.set_attribute( + SpanAttributes.RECORD_ROOT.APP_VERSION, self.app_version + ) + root_span.set_attribute(SpanAttributes.RECORD_ROOT.APP_ID, self.app_id) + root_span.set_attribute( + SpanAttributes.RECORD_ROOT.RECORD_ID, otel_record_id + ) + + return root_span + def __exit__(self, exc_type, exc_value, exc_tb): - print("exit") + remove_baggage(SpanAttributes.RECORD_ID) + logging.debug("Exiting the OTEL app context.") + + if self.token: + # Clearing the context once we're done with this root span. + # See https://github.com/open-telemetry/opentelemetry-python/issues/2432#issuecomment-1593458684 + context_api.detach(self.token) + + if self.span_context: + self.span_context.__exit__(exc_type, exc_value, exc_tb) diff --git a/src/otel/semconv/trulens/otel/semconv/trace.py b/src/otel/semconv/trulens/otel/semconv/trace.py index 9050f81ae..e6ffa384f 100644 --- a/src/otel/semconv/trulens/otel/semconv/trace.py +++ b/src/otel/semconv/trulens/otel/semconv/trace.py @@ -55,6 +55,13 @@ class SpanAttributes: """ User-defined selector name for the current span. """ + SPAN_TYPE = "trulens.span_type" + """Key for the span type attribute.""" + + RECORD_ID = "trulens.record_id" + """ID of the record that the span belongs to.""" + + SPAN_TYPES = "trulens.span_types" class SpanType(str, Enum): """Span type attribute values. From 41b37b0399186220d85d85759fbe5f4871ca2fa9 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Thu, 19 Dec 2024 15:33:19 -0500 Subject: [PATCH 49/59] update --- src/core/trulens/experimental/otel_tracing/core/instrument.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 56947f14a..18f758b1c 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -12,7 +12,6 @@ from opentelemetry.baggage import remove_baggage from opentelemetry.baggage import set_baggage import opentelemetry.context as context_api -from trulens.apps.custom import instrument as custom_instrument from trulens.core import app as core_app from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME <<<<<<< HEAD From 206790924df5c9932f0b826aefc8c588fffc366c Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 12:42:48 -0500 Subject: [PATCH 50/59] save --- examples/experimental/otel_exporter.ipynb | 39 +---------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 67b972b42..3cb2adabc 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -165,26 +165,6 @@ " sys.path.append(str(base_dir))" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "\n", - "root = logging.getLogger()\n", - "root.setLevel(logging.DEBUG)\n", - "handler = logging.StreamHandler(sys.stdout)\n", - "handler.setLevel(logging.DEBUG)\n", - "handler.addFilter(logging.Filter(\"trulens\"))\n", - "formatter = logging.Formatter(\n", - " \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n", - ")\n", - "handler.setFormatter(formatter)\n", - "root.addHandler(handler)" - ] - }, { "cell_type": "code", "execution_count": null, @@ -219,30 +199,13 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "\n", "import dotenv\n", - "from trulens.connectors.snowflake import SnowflakeConnector\n", "from trulens.core.session import TruSession\n", "from trulens.experimental.otel_tracing.core.init import init\n", "\n", "dotenv.load_dotenv()\n", "\n", - "connection_params = {\n", - " \"account\": os.environ[\"SNOWFLAKE_ACCOUNT\"],\n", - " \"user\": os.environ[\"SNOWFLAKE_USER\"],\n", - " \"password\": os.environ[\"SNOWFLAKE_USER_PASSWORD\"],\n", - " \"database\": os.environ[\"SNOWFLAKE_DATABASE\"],\n", - " \"schema\": os.environ[\"SNOWFLAKE_SCHEMA\"],\n", - " \"warehouse\": os.environ[\"SNOWFLAKE_WAREHOUSE\"],\n", - " \"role\": os.environ[\"SNOWFLAKE_ROLE\"],\n", - "}\n", - "\n", - "connector = SnowflakeConnector(\n", - " **connection_params, database_redact_keys=True, database_args=None\n", - ")\n", - "session = TruSession(connector=connector)\n", - "session.experimental_enable_feature(\"otel_tracing\")\n", + "session = TruSession()\n", "session.reset_database()\n", "init(session, debug=True)" ] From c61dda6f884cd9068e5eb4443f54c64dbe0c7875 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 12:50:31 -0500 Subject: [PATCH 51/59] add back debugger --- examples/experimental/otel_exporter.ipynb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 3cb2adabc..62f492a9f 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -165,6 +165,26 @@ " sys.path.append(str(base_dir))" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "root = logging.getLogger()\n", + "root.setLevel(logging.DEBUG)\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setLevel(logging.DEBUG)\n", + "handler.addFilter(logging.Filter(\"trulens\"))\n", + "formatter = logging.Formatter(\n", + " \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n", + ")\n", + "handler.setFormatter(formatter)\n", + "root.addHandler(handler)" + ] + }, { "cell_type": "code", "execution_count": null, From 402c0e5de220e457342a2bfd1fde67707d50f55e Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 12:56:45 -0500 Subject: [PATCH 52/59] update notebook --- examples/experimental/otel_exporter.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index 62f492a9f..bb75e1d03 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -226,6 +226,7 @@ "dotenv.load_dotenv()\n", "\n", "session = TruSession()\n", + "session.experimental_enable_feature(\"otel_tracing\")\n", "session.reset_database()\n", "init(session, debug=True)" ] From 364f6aadd4a7f6e580a3ae6f42539226aa0ae89f Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 19:28:19 -0500 Subject: [PATCH 53/59] fix --- src/otel/semconv/trulens/otel/semconv/trace.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/otel/semconv/trulens/otel/semconv/trace.py b/src/otel/semconv/trulens/otel/semconv/trace.py index e6ffa384f..13c59a247 100644 --- a/src/otel/semconv/trulens/otel/semconv/trace.py +++ b/src/otel/semconv/trulens/otel/semconv/trace.py @@ -39,11 +39,6 @@ class SpanAttributes: Base prefix for the other keys. """ - SPAN_TYPE = BASE + "span_type" - """ - Span type attribute. - """ - SELECTOR_NAME_KEY = "selector_name" """ Key for the user-defined selector name for the current span. From 17619e652fb63d9fe238d3c2a16410213417fa68 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Fri, 20 Dec 2024 19:30:40 -0500 Subject: [PATCH 54/59] update semcov --- src/otel/semconv/trulens/otel/semconv/trace.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/otel/semconv/trulens/otel/semconv/trace.py b/src/otel/semconv/trulens/otel/semconv/trace.py index 13c59a247..85dff9ca5 100644 --- a/src/otel/semconv/trulens/otel/semconv/trace.py +++ b/src/otel/semconv/trulens/otel/semconv/trace.py @@ -39,6 +39,11 @@ class SpanAttributes: Base prefix for the other keys. """ + SPAN_TYPE = BASE + "span_type" + """ + Span type attribute. + """ + SELECTOR_NAME_KEY = "selector_name" """ Key for the user-defined selector name for the current span. @@ -50,13 +55,11 @@ class SpanAttributes: """ User-defined selector name for the current span. """ - SPAN_TYPE = "trulens.span_type" - """Key for the span type attribute.""" - RECORD_ID = "trulens.record_id" + RECORD_ID = BASE + "record_id" """ID of the record that the span belongs to.""" - SPAN_TYPES = "trulens.span_types" + SPAN_TYPES = BASE + "span_types" class SpanType(str, Enum): """Span type attribute values. From d300f671a7d9f1e9b76b2029f11b68ef99bdab52 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Mon, 23 Dec 2024 21:08:52 -0500 Subject: [PATCH 55/59] remove artifacts --- examples/experimental/otel_exporter.ipynb | 188 ++++-------------- .../otel_tracing/core/instrument.py | 20 +- 2 files changed, 38 insertions(+), 170 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index bb75e1d03..a3a145b3e 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -1,153 +1,14 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install opentelemetry-api\n", - "# !pip install opentelemetry-sdk" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import sys\n", - "\n", - "# Add base dir to path to be able to access test folder.\n", - "base_dir = Path().cwd().parent.parent.resolve()\n", - "if str(base_dir) not in sys.path:\n", - " print(f\"Adding {base_dir} to sys.path\")\n", - " sys.path.append(str(base_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "\n", - "root = logging.getLogger()\n", - "root.setLevel(logging.DEBUG)\n", - "handler = logging.StreamHandler(sys.stdout)\n", - "handler.setLevel(logging.DEBUG)\n", - "handler.addFilter(logging.Filter(\"trulens\"))\n", - "formatter = logging.Formatter(\n", - " \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n", - ")\n", - "handler.setFormatter(formatter)\n", - "root.addHandler(handler)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.experimental.otel_tracing.core.instrument import instrument\n", - "\n", - "\n", - "class TestApp:\n", - " @instrument()\n", - " def respond_to_query(self, query: str) -> str:\n", - " return f\"answer: {self.nested(query)}\"\n", - "\n", - " @instrument(attributes={\"nested_attr1\": \"value1\"})\n", - " def nested(self, query: str) -> str:\n", - " return f\"nested: {self.nested2(query)}\"\n", - "\n", - " @instrument(\n", - " attributes=lambda ret, exception, *args, **kwargs: {\n", - " \"nested2_ret\": ret,\n", - " \"nested2_args[0]\": args[0],\n", - " }\n", - " )\n", - " def nested2(self, query: str) -> str:\n", - " nested_result = \"\"\n", - "\n", - " try:\n", - " nested_result = self.nested3(query)\n", - " except Exception:\n", - " pass\n", - "\n", - " return f\"nested2: {nested_result}\"\n", - "\n", - " @instrument(\n", - " attributes=lambda ret, exception, *args, **kwargs: {\n", - " \"nested3_ex\": exception.args if exception else None,\n", - " \"nested3_ret\": ret,\n", - " \"selector_name\": \"special\",\n", - " \"cows\": \"moo\",\n", - " }\n", - " )\n", - " def nested3(self, query: str) -> str:\n", - " if query == \"throw\":\n", - " raise ValueError(\"nested3 exception\")\n", - " return \"nested3\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import dotenv\n", - "from trulens.core.session import TruSession\n", - "from trulens.experimental.otel_tracing.core.init import init\n", - "\n", - "dotenv.load_dotenv()\n", - "\n", - "session = TruSession()\n", - "session.experimental_enable_feature(\"otel_tracing\")\n", - "session.reset_database()\n", - "init(session, debug=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens.apps.custom import TruCustomApp\n", - "\n", - "test_app = TestApp()\n", - "custom_app = TruCustomApp(test_app)\n", - "\n", - "with custom_app as recording:\n", - " test_app.respond_to_query(\"test\")\n", - "\n", - "with custom_app as recording:\n", - " test_app.respond_to_query(\"throw\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "trulens", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install opentelemetry-api\n", + "# !pip install opentelemetry-sdk" + ] }, { "cell_type": "code", @@ -204,13 +65,33 @@ " return f\"nested: {self.nested2(query)}\"\n", "\n", " @instrument(\n", - " attributes=lambda ret, *args, **kwargs: {\n", + " attributes=lambda ret, exception, *args, **kwargs: {\n", " \"nested2_ret\": ret,\n", " \"nested2_args[0]\": args[0],\n", " }\n", " )\n", " def nested2(self, query: str) -> str:\n", - " return f\"nested2: {query}\"" + " nested_result = \"\"\n", + "\n", + " try:\n", + " nested_result = self.nested3(query)\n", + " except Exception:\n", + " pass\n", + "\n", + " return f\"nested2: {nested_result}\"\n", + "\n", + " @instrument(\n", + " attributes=lambda ret, exception, *args, **kwargs: {\n", + " \"nested3_ex\": exception.args if exception else None,\n", + " \"nested3_ret\": ret,\n", + " \"selector_name\": \"special\",\n", + " \"cows\": \"moo\",\n", + " }\n", + " )\n", + " def nested3(self, query: str) -> str:\n", + " if query == \"throw\":\n", + " raise ValueError(\"nested3 exception\")\n", + " return \"nested3\"" ] }, { @@ -243,7 +124,10 @@ "custom_app = TruCustomApp(test_app)\n", "\n", "with custom_app as recording:\n", - " test_app.respond_to_query(\"test\")" + " test_app.respond_to_query(\"test\")\n", + "\n", + "with custom_app as recording:\n", + " test_app.respond_to_query(\"throw\")" ] } ], diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 18f758b1c..e5e7d33cc 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -1,11 +1,7 @@ from functools import wraps import logging -<<<<<<< HEAD from typing import Any, Callable, Dict, Optional, Union -======= -from typing import Any, Callable, Optional, Union import uuid ->>>>>>> ae0e4d895 (draft) from opentelemetry import trace from opentelemetry.baggage import get_baggage @@ -14,12 +10,6 @@ import opentelemetry.context as context_api from trulens.core import app as core_app from trulens.experimental.otel_tracing.core.init import TRULENS_SERVICE_NAME -<<<<<<< HEAD -======= -from trulens.experimental.otel_tracing.core.semantic import ( - TRULENS_SELECTOR_NAME, -) ->>>>>>> 41ef4d524 (draft) from trulens.otel.semconv.trace import SpanAttributes logger = logging.getLogger(__name__) @@ -93,21 +83,17 @@ def wrapper(*args, **kwargs): func_exception: Optional[Exception] = None attributes_exception: Optional[Exception] = None -<<<<<<< HEAD try: ret = func(*args, **kwargs) except Exception as e: # We want to get into the next clause to allow the users to still add attributes. # It's on the user to deal with None as a return value. -<<<<<<< HEAD func_exception = e -======= - exception = e -======= + span.set_attribute("name", func.__name__) span.set_attribute("kind", "SPAN_KIND_TRULENS") span.set_attribute( - "parent_span_id", parent_span.get_span_context().span_id + "parent_span_id", span.get_span_context().span_id ) span.set_attribute( SpanAttributes.RECORD_ID, @@ -115,8 +101,6 @@ def wrapper(*args, **kwargs): ) ret = func(*args, **kwargs) ->>>>>>> 41ef4d524 (draft) ->>>>>>> ae0e4d895 (draft) try: attributes_to_add = {} From bc26f34d0da3fb2272e977fedaaea7d5817fc601 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Mon, 23 Dec 2024 21:17:06 -0500 Subject: [PATCH 56/59] remove span_types from SpanAttributes --- src/otel/semconv/trulens/otel/semconv/trace.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/otel/semconv/trulens/otel/semconv/trace.py b/src/otel/semconv/trulens/otel/semconv/trace.py index 85dff9ca5..6200ef668 100644 --- a/src/otel/semconv/trulens/otel/semconv/trace.py +++ b/src/otel/semconv/trulens/otel/semconv/trace.py @@ -59,8 +59,6 @@ class SpanAttributes: RECORD_ID = BASE + "record_id" """ID of the record that the span belongs to.""" - SPAN_TYPES = BASE + "span_types" - class SpanType(str, Enum): """Span type attribute values. From 86850df56736630ee1e43ecd8ca467104bb34063 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Mon, 23 Dec 2024 21:35:49 -0500 Subject: [PATCH 57/59] modified it to accept multiple tokens --- src/core/trulens/core/app.py | 5 +++-- .../otel_tracing/core/instrument.py | 22 +++++++++++++++---- .../semconv/trulens/otel/semconv/trace.py | 3 +++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/core/trulens/core/app.py b/src/core/trulens/core/app.py index 7fc316a8c..0a6576321 100644 --- a/src/core/trulens/core/app.py +++ b/src/core/trulens/core/app.py @@ -422,9 +422,10 @@ def tru(self) -> core_connector.DBConnector: pydantic.PrivateAttr(default_factory=dict) ) - token: Optional[object] = None + tokens: list[object] = [] """ - OTEL context token for the current context manager. + OTEL context tokens for the current context manager. These tokens are how the OTEL + context api keeps track of what is changed in the context, and used to undo the changes. """ span_context: Optional[contextlib.AbstractContextManager] = None diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index e5e7d33cc..7978be8a7 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -99,6 +99,10 @@ def wrapper(*args, **kwargs): SpanAttributes.RECORD_ID, str(get_baggage(SpanAttributes.RECORD_ID)), ) + span.set_attribute( + SpanAttributes.APP_ID, + str(get_baggage(SpanAttributes.APP_ID)), + ) ret = func(*args, **kwargs) @@ -170,9 +174,14 @@ def __enter__(self): # Calling set_baggage does not actually add the baggage to the current context, but returns a new one # To avoid issues with remembering to add/remove the baggage, we attach it to the runtime context. - self.token = context_api.attach( - set_baggage(SpanAttributes.RECORD_ID, otel_record_id) + self.tokens.append( + context_api.attach( + set_baggage(SpanAttributes.RECORD_ID, otel_record_id) + ) ) + # self.tokens.append(context_api.attach( + # set_baggage(SpanAttributes.APP_ID, self.app_id) + # )) # Use start_as_current_span as a context manager self.span_context = tracer.start_as_current_span("root") @@ -180,11 +189,16 @@ def __enter__(self): logger.debug(str(get_baggage(SpanAttributes.RECORD_ID))) + # Set general span attributes root_span.set_attribute("kind", "SPAN_KIND_TRULENS") root_span.set_attribute("name", "root") root_span.set_attribute( SpanAttributes.SPAN_TYPE, SpanAttributes.SpanType.RECORD_ROOT ) + root_span.set_attribute(SpanAttributes.APP_ID, self.app_id) + root_span.set_attribute(SpanAttributes.RECORD_ID, otel_record_id) + + # Set record root specific attributes root_span.set_attribute( SpanAttributes.RECORD_ROOT.APP_NAME, self.app_name ) @@ -202,10 +216,10 @@ def __exit__(self, exc_type, exc_value, exc_tb): remove_baggage(SpanAttributes.RECORD_ID) logging.debug("Exiting the OTEL app context.") - if self.token: + while len(self.tokens) > 0: # Clearing the context once we're done with this root span. # See https://github.com/open-telemetry/opentelemetry-python/issues/2432#issuecomment-1593458684 - context_api.detach(self.token) + context_api.detach(self.tokens.pop()) if self.span_context: self.span_context.__exit__(exc_type, exc_value, exc_tb) diff --git a/src/otel/semconv/trulens/otel/semconv/trace.py b/src/otel/semconv/trulens/otel/semconv/trace.py index 6200ef668..3175e6bff 100644 --- a/src/otel/semconv/trulens/otel/semconv/trace.py +++ b/src/otel/semconv/trulens/otel/semconv/trace.py @@ -59,6 +59,9 @@ class SpanAttributes: RECORD_ID = BASE + "record_id" """ID of the record that the span belongs to.""" + APP_ID = BASE + "app_id" + """ID of the app that the span belongs to.""" + class SpanType(str, Enum): """Span type attribute values. From 57b68091726f400fc5ef0d5063aafb269f71c997 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Mon, 23 Dec 2024 21:53:41 -0500 Subject: [PATCH 58/59] fix bug with multiple func calls --- examples/experimental/otel_exporter.ipynb | 1 + .../experimental/otel_tracing/core/instrument.py | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/experimental/otel_exporter.ipynb b/examples/experimental/otel_exporter.ipynb index a3a145b3e..bc57df4ed 100644 --- a/examples/experimental/otel_exporter.ipynb +++ b/examples/experimental/otel_exporter.ipynb @@ -109,6 +109,7 @@ "session = TruSession()\n", "session.experimental_enable_feature(\"otel_tracing\")\n", "session.reset_database()\n", + "\n", "init(session, debug=True)" ] }, diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index 7978be8a7..dcd8b9945 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -104,8 +104,6 @@ def wrapper(*args, **kwargs): str(get_baggage(SpanAttributes.APP_ID)), ) - ret = func(*args, **kwargs) - try: attributes_to_add = {} @@ -164,7 +162,7 @@ def wrapper(*args, **kwargs): class App(core_app.App): # For use as a context manager. def __enter__(self): - logging.debug("Entering the OTEL app context.") + logger.debug("Entering the OTEL app context.") # Note: This is not the same as the record_id in the core app since the OTEL # tracing is currently separate from the old records behavior @@ -179,9 +177,9 @@ def __enter__(self): set_baggage(SpanAttributes.RECORD_ID, otel_record_id) ) ) - # self.tokens.append(context_api.attach( - # set_baggage(SpanAttributes.APP_ID, self.app_id) - # )) + self.tokens.append( + context_api.attach(set_baggage(SpanAttributes.APP_ID, self.app_id)) + ) # Use start_as_current_span as a context manager self.span_context = tracer.start_as_current_span("root") @@ -214,7 +212,9 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_tb): remove_baggage(SpanAttributes.RECORD_ID) - logging.debug("Exiting the OTEL app context.") + remove_baggage(SpanAttributes.APP_ID) + + logger.debug("Exiting the OTEL app context.") while len(self.tokens) > 0: # Clearing the context once we're done with this root span. From 14869e17aa19e869081110b337abe2ead9503ff0 Mon Sep 17 00:00:00 2001 From: Garett Tok Ern Liang Date: Tue, 24 Dec 2024 10:17:55 -0500 Subject: [PATCH 59/59] PR feedback --- src/core/trulens/core/app.py | 2 +- src/core/trulens/experimental/otel_tracing/core/instrument.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/trulens/core/app.py b/src/core/trulens/core/app.py index 0a6576321..a756208df 100644 --- a/src/core/trulens/core/app.py +++ b/src/core/trulens/core/app.py @@ -422,7 +422,7 @@ def tru(self) -> core_connector.DBConnector: pydantic.PrivateAttr(default_factory=dict) ) - tokens: list[object] = [] + tokens: List[object] = [] """ OTEL context tokens for the current context manager. These tokens are how the OTEL context api keeps track of what is changed in the context, and used to undo the changes. diff --git a/src/core/trulens/experimental/otel_tracing/core/instrument.py b/src/core/trulens/experimental/otel_tracing/core/instrument.py index dcd8b9945..d114adc79 100644 --- a/src/core/trulens/experimental/otel_tracing/core/instrument.py +++ b/src/core/trulens/experimental/otel_tracing/core/instrument.py @@ -216,7 +216,7 @@ def __exit__(self, exc_type, exc_value, exc_tb): logger.debug("Exiting the OTEL app context.") - while len(self.tokens) > 0: + while self.tokens: # Clearing the context once we're done with this root span. # See https://github.com/open-telemetry/opentelemetry-python/issues/2432#issuecomment-1593458684 context_api.detach(self.tokens.pop())