Skip to content

Commit

Permalink
adds reporting of response time to explorer chat messages (microsoft#243
Browse files Browse the repository at this point in the history
)

Shows up in the footer of the chat message:

![image](https://github.com/user-attachments/assets/e7822513-139c-4387-a935-4caf82bb2f21)
  • Loading branch information
bkrabach authored Nov 13, 2024
1 parent 29aa96a commit 29e5e78
Showing 1 changed file with 22 additions and 1 deletion.
23 changes: 22 additions & 1 deletion assistants/explorer-assistant/assistant/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import logging
import re
import time
from typing import Any, Awaitable, Callable

import deepmerge
Expand Down Expand Up @@ -183,6 +184,9 @@ async def respond_to_conversation(
# define the metadata key for any metadata created within this method
method_metadata_key = "respond_to_conversation"

# track the start time of the response generation
response_start_time = time.time()

# get the list of conversation participants
participants_response = await context.get_participants(include_inactive=True)

Expand Down Expand Up @@ -371,13 +375,22 @@ async def respond_to_conversation(
},
)

# create the footer items for the response
footer_items = []

if completion is not None:
# get the total tokens used for the completion
completion_total_tokens = completion.usage.total_tokens if completion.usage else 0
footer_items.append(_get_token_usage_message(config.request_config.max_tokens, completion_total_tokens))

# add the completion to the metadata for debugging
# track the end time of the response generation
response_end_time = time.time()
response_duration = response_end_time - response_start_time

# add the response duration to the footer items
footer_items.append(_get_response_duration_message(response_duration))

# update the metadata with debug information
deepmerge.always_merger.merge(
metadata,
{
Expand Down Expand Up @@ -593,6 +606,14 @@ async def _get_history_messages(
return history


def _get_response_duration_message(response_duration: float) -> str:
"""
Generate a display friendly message for the response duration, to be added to the footer items.
"""

return f"Response time: {response_duration:.2f} seconds"


def _get_token_usage_message(
max_tokens: int,
completion_total_tokens: int,
Expand Down

0 comments on commit 29e5e78

Please sign in to comment.