Add support for GenericMessageContent to AmazonBedrockPromptDriver an…

…d AnthropicPromptDriver

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support for `BranchTask` in `StructureVisualizer`.
 - `EvalEngine` for evaluating the performance of an LLM's output against a given input.
 - `BaseFileLoader.save()` method for saving an Artifact to a destination.
+- Support for `GenericMessageContent` in `AnthropicPromptDriver` and `AmazonBedrockPromptDriver`.
 
 ### Changed
 

docs/examples/src/talk_to_a_document.py

@@ -0,0 +1,42 @@
+import base64
+
+import requests
+from attrs import define
+
+from griptape.artifacts import GenericArtifact, TextArtifact
+from griptape.drivers import AnthropicPromptDriver
+from griptape.structures import Agent
+
+
+@define
+class DocumentArtifact(GenericArtifact):
+    """Artifact for storing a document.
+
+    Subclassing `GenericArtifact` to avoid printing out `self.value` as `GenericArtifact` does.
+    """
+
+    def to_text(self) -> str:
+        return f"Document: {self.value['source']['media_type']}"
+
+
+doc_bytes = requests.get("https://arxiv.org/pdf/1706.03762.pdf").content
+
+
+agent = Agent(
+    prompt_driver=AnthropicPromptDriver(model="claude-3-5-sonnet-20240620", max_attempts=0),
+    input=[
+        DocumentArtifact(
+            {
+                "type": "document",
+                "source": {
+                    "type": "base64",
+                    "media_type": "application/pdf",
+                    "data": base64.b64encode(doc_bytes).decode("utf-8"),
+                },
+            }
+        ),
+        TextArtifact("{{ args[0] }}"),
+    ],
+)
+
+agent.run("What is the title and who are the authors of this paper?")

docs/examples/talk-to-a-document.md

@@ -0,0 +1,22 @@
+Some LLM providers such as [Anthropic](https://docs.anthropic.com/en/api/messages#body-messages-content) and [Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_DocumentBlock.html) offer the ability to pass documents directly to the LLM.
+
+In this example, we create a custom `DocumentArtifact` to pass a PDF document to the Agent. The Agent then uses the document to answer questions about the paper.
+
+```python
+--8<-- "docs/examples/src/talk_to_a_document.py"
+```
+
+```
+[12/23/24 09:37:47] INFO     PromptTask cc77e4c193c84a5986a4e02e56614d6b
+                             Input: Document: application/pdf
+
+                             What is the title and who are the authors of this paper?
+[12/23/24 09:37:57] INFO     PromptTask cc77e4c193c84a5986a4e02e56614d6b
+                             Output: The title of this paper is "Attention Is All You Need" and the authors are:
+
+                             Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Łukasz Kaiser, and Illia
+                             Polosukhin.
+
+                             The paper is from Google Brain, Google Research, and the University of Toronto. It introduces the Transformer model
+                             architecture for sequence transduction tasks like machine translation.
+```

griptape/drivers/prompt/amazon_bedrock_prompt_driver.py

@@ -188,7 +188,7 @@ def __to_bedrock_message_content(self, content: BaseMessageContent) -> dict:
                 },
             }
         else:
-            raise ValueError(f"Unsupported content type: {type(content)}")
+            return content.artifact.value
 
     def __to_bedrock_tool_use_content(self, artifact: BaseArtifact) -> dict:
         if isinstance(artifact, ImageArtifact):

griptape/drivers/prompt/anthropic_prompt_driver.py

@@ -192,7 +192,7 @@ def __to_anthropic_message_content(self, content: BaseMessageContent) -> dict:
                 "is_error": isinstance(artifact, ErrorArtifact),
             }
         else:
-            raise ValueError(f"Unsupported prompt content type: {type(content)}")
+            return content.artifact.value
 
     def __to_anthropic_tool_result_content(self, artifact: BaseArtifact) -> dict:
         if isinstance(artifact, ImageArtifact):

mkdocs.yml

@@ -176,6 +176,7 @@ nav:
           - Talk to a PDF: "examples/talk-to-a-pdf.md"
           - Talk to a Video: "examples/talk-to-a-video.md"
           - Talk to an Image: "examples/talk-to-an-image.md"
+          - Talk to a Document: "examples/talk-to-a-document.md"
           - Multi Agent Workflows: "examples/multi-agent-workflow.md"
           - Shared Memory Between Agents: "examples/multiple-agent-shared-memory.md"
           - Chat Sessions with Amazon DynamoDB: "examples/amazon-dynamodb-sessions.md"

tests/unit/drivers/prompt/test_amazon_bedrock_prompt_driver.py

@@ -1,6 +1,6 @@
 import pytest
 
-from griptape.artifacts import ActionArtifact, ErrorArtifact, ImageArtifact, ListArtifact, TextArtifact
+from griptape.artifacts import ActionArtifact, ErrorArtifact, GenericArtifact, ImageArtifact, ListArtifact, TextArtifact
 from griptape.common import ActionCallDeltaMessageContent, PromptStack, TextDeltaMessageContent, ToolAction
 from griptape.drivers import AmazonBedrockPromptDriver
 from tests.mocks.mock_tool.tool import MockTool
@@ -299,6 +299,7 @@ def prompt_stack(self, request):
                 ]
             )
         )
+        prompt_stack.add_user_message(GenericArtifact("video-file"))
 
         return prompt_stack
 
@@ -354,6 +355,7 @@ def messages(self):
                 ],
                 "role": "user",
             },
+            {"content": ["video-file"], "role": "user"},
         ]
 
     @pytest.mark.parametrize("use_native_tools", [True, False])

tests/unit/drivers/prompt/test_anthropic_prompt_driver.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from griptape.artifacts import ActionArtifact, ImageArtifact, ListArtifact, TextArtifact
+from griptape.artifacts import ActionArtifact, GenericArtifact, ImageArtifact, ListArtifact, TextArtifact
 from griptape.artifacts.error_artifact import ErrorArtifact
 from griptape.common import ActionCallDeltaMessageContent, PromptStack, TextDeltaMessageContent, ToolAction
 from griptape.drivers import AnthropicPromptDriver
@@ -270,6 +270,7 @@ def prompt_stack(self, request):
                 ]
             )
         )
+        prompt_stack.add_user_message(GenericArtifact({"type": "document"}))
 
         return prompt_stack
 
@@ -337,6 +338,12 @@ def messages(self):
                 ],
                 "role": "user",
             },
+            {
+                "content": [
+                    {"type": "document"},
+                ],
+                "role": "user",
+            },
         ]
 
     def test_init(self):