Merge branch 'dev' into feature/exa_web_search_driver

CHANGELOG.md

@@ -35,6 +35,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **BREAKING**: Renamed parameter `pipe` on `HuggingFacePipelinePromptDriver` to `pipeline`.
 - Several places where API clients are initialized are now lazy loaded.
 - `Structure.output`'s type is now `BaseArtifact` and raises an exception if the output is `None`.
+- **BREAKING**: Update `pypdf` dependency to `^5.0.1`.
+- **BREAKING**: Update `redis` dependency to `^5.1.0`.
+- `MarkdownifyWebScraperDriver.DEFAULT_EXCLUDE_TAGS` now includes media/blob-like HTML tags
+
+### Fixed
+- Anthropic native Tool calling
 
 ## [0.32.0] - 2024-09-17
 

griptape/drivers/memory/conversation/redis_conversation_memory_driver.py

@@ -58,5 +58,5 @@ def store(self, runs: list[Run], metadata: dict[str, Any]) -> None:
     def load(self) -> tuple[list[Run], dict[str, Any]]:
         memory_json = self.client.hget(self.index, self.conversation_id)
         if memory_json is not None:
-            return self._from_params_dict(json.loads(memory_json))
+            return self._from_params_dict(json.loads(memory_json))  # pyright: ignore[reportArgumentType] https://github.com/redis/redis-py/issues/2399
         return [], {}

griptape/drivers/prompt/anthropic_prompt_driver.py

@@ -127,7 +127,7 @@ def __to_anthropic_role(self, message: Message) -> str:
             return "user"
 
     def __to_anthropic_tools(self, tools: list[BaseTool]) -> list[dict]:
-        return [
+        tool_schemas = [
             {
                 "name": tool.to_native_tool_name(activity),
                 "description": tool.activity_description(activity),
@@ -137,6 +137,13 @@ def __to_anthropic_tools(self, tools: list[BaseTool]) -> list[dict]:
             for activity in tool.activities()
         ]
 
+        # Anthropic doesn't support $schema and $id
+        for tool_schema in tool_schemas:
+            del tool_schema["input_schema"]["$schema"]
+            del tool_schema["input_schema"]["$id"]
+
+        return tool_schemas
+
     def __to_anthropic_content(self, message: Message) -> str | list[dict]:
         if message.has_all_content_type(TextMessageContent):
             return message.to_text()

griptape/drivers/web_scraper/markdownify_web_scraper_driver.py

@@ -27,7 +27,7 @@ class MarkdownifyWebScraperDriver(BaseWebScraperDriver):
             the browser has emitted the "load" event.
     """
 
-    DEFAULT_EXCLUDE_TAGS = ["script", "style", "head"]
+    DEFAULT_EXCLUDE_TAGS = ["script", "style", "head", "audio", "img", "picture", "source", "video"]
 
     include_links: bool = field(default=True, kw_only=True)
     exclude_tags: list[str] = field(

pyproject.toml

@@ -38,7 +38,7 @@ snowflake-sqlalchemy = { version = "^1.6.1", optional = true }
 pinecone-client = { version = "^3", optional = true }
 pymongo = { version = "^4.8.0", optional = true }
 marqo = { version = "^3.7.0", optional = true }
-redis = { version = "^4.6.0", optional = true }
+redis = { version = "^5.1.0", optional = true }
 opensearch-py = { version = "^2.3.1", optional = true }
 pgvector = { version = ">=0.2.3,<0.4.0", optional = true }
 psycopg2-binary = { version = "^2.9.9", optional = true }
@@ -69,7 +69,7 @@ exa-py = {version = "^1.1.4", optional = true}
 
 # loaders
 pandas = {version = "^1.3", optional = true}
-pypdf = {version = "^3.9", optional = true}
+pypdf = {version = "^5.0.1", optional = true}
 pillow = {version = "^10.2.0", optional = true}
 mail-parser = {version = "^3.15.0", optional = true}
 filetype = {version = "^1.2", optional = true}
@@ -218,7 +218,7 @@ pytest-mock = "^3.1.4"
 mongomock = "^4.1.2"
 
 twine = "^5.1.1"
-moto = {extras = ["dynamodb", "iotdata", "sqs"], version = "^4.2.13"}
+moto = {extras = ["dynamodb", "iotdata", "sqs"], version = "^5.0.16"}
 pytest-xdist = "^3.3.1"
 pytest-cov = "^5.0.0"
 pytest-env = "^1.1.1"

tests/unit/drivers/event_listener/test_amazon_sqs_event_listener_driver.py

@@ -1,6 +1,6 @@
 import boto3
 import pytest
-from moto import mock_sqs
+from moto import mock_aws
 
 from griptape.drivers.event_listener.amazon_sqs_event_listener_driver import AmazonSqsEventListenerDriver
 from tests.mocks.mock_event import MockEvent
@@ -14,7 +14,7 @@ def _run_before_and_after_tests(self):
 
     @pytest.fixture()
     def driver(self):
-        mock = mock_sqs()
+        mock = mock_aws()
         mock.start()
 
         session = boto3.Session(region_name="us-east-1")

tests/unit/drivers/event_listener/test_aws_iot_event_listener_driver.py

@@ -1,13 +1,13 @@
 import boto3
 import pytest
-from moto import mock_iotdata
+from moto import mock_aws
 
 from griptape.drivers.event_listener.aws_iot_core_event_listener_driver import AwsIotCoreEventListenerDriver
 from tests.mocks.mock_event import MockEvent
 from tests.utils.aws import mock_aws_credentials
 
 
-@mock_iotdata
+@mock_aws
 class TestAwsIotCoreEventListenerDriver:
     @pytest.fixture(autouse=True)
     def _run_before_and_after_tests(self):

tests/unit/drivers/file_manager/test_amazon_s3_file_manager_driver.py

@@ -3,7 +3,7 @@
 
 import boto3
 import pytest
-from moto import mock_s3
+from moto import mock_aws
 
 from griptape.artifacts import InfoArtifact, ListArtifact, TextArtifact
 from griptape.drivers import AmazonS3FileManagerDriver
@@ -18,7 +18,7 @@ def _set_aws_credentials(self):
 
     @pytest.fixture()
     def session(self):
-        mock = mock_s3()
+        mock = mock_aws()
         mock.start()
         yield boto3.Session(region_name="us-east-1")
         mock.stop()

tests/unit/drivers/memory/conversation/test_dynamodb_conversation_memory_driver.py

@@ -1,6 +1,6 @@
 import boto3
 import pytest
-from moto import mock_dynamodb
+from moto import mock_aws
 
 from griptape.drivers import AmazonDynamoDbConversationMemoryDriver
 from griptape.memory.structure import ConversationMemory
@@ -11,15 +11,18 @@
 
 class TestDynamoDbConversationMemoryDriver:
     DYNAMODB_TABLE_NAME = "griptape"
+    DYNAMODB_COMPOSITE_TABLE_NAME = "griptape_composite"
     DYNAMODB_PARTITION_KEY = "entryId"
+    DYNAMODB_SORT_KEY = "sortKey"
     AWS_REGION = "us-west-2"
     VALUE_ATTRIBUTE_KEY = "foo"
     PARTITION_KEY_VALUE = "bar"
+    SORT_KEY_VALUE = "baz"
 
     @pytest.fixture(autouse=True)
     def _run_before_and_after_tests(self):
         mock_aws_credentials()
-        self.mock_dynamodb = mock_dynamodb()
+        self.mock_dynamodb = mock_aws()
         self.mock_dynamodb.start()
 
         dynamodb = boto3.Session(region_name=self.AWS_REGION).client("dynamodb")
@@ -30,9 +33,23 @@ def _run_before_and_after_tests(self):
             BillingMode="PAY_PER_REQUEST",
         )
 
+        dynamodb.create_table(
+            TableName=self.DYNAMODB_COMPOSITE_TABLE_NAME,
+            KeySchema=[
+                {"AttributeName": self.DYNAMODB_PARTITION_KEY, "KeyType": "HASH"},
+                {"AttributeName": self.DYNAMODB_SORT_KEY, "KeyType": "RANGE"},
+            ],
+            AttributeDefinitions=[
+                {"AttributeName": self.DYNAMODB_PARTITION_KEY, "AttributeType": "S"},
+                {"AttributeName": self.DYNAMODB_SORT_KEY, "AttributeType": "S"},
+            ],
+            BillingMode="PAY_PER_REQUEST",
+        )
+
         yield
 
         dynamodb.delete_table(TableName=self.DYNAMODB_TABLE_NAME)
+        dynamodb.delete_table(TableName=self.DYNAMODB_COMPOSITE_TABLE_NAME)
         self.mock_dynamodb.stop()
 
     def test_store(self):
@@ -62,27 +79,31 @@ def test_store(self):
     def test_store_with_sort_key(self):
         session = boto3.Session(region_name=self.AWS_REGION)
         dynamodb = session.resource("dynamodb")
-        table = dynamodb.Table(self.DYNAMODB_TABLE_NAME)
+        table = dynamodb.Table(self.DYNAMODB_COMPOSITE_TABLE_NAME)
         memory_driver = AmazonDynamoDbConversationMemoryDriver(
             session=session,
-            table_name=self.DYNAMODB_TABLE_NAME,
+            table_name=self.DYNAMODB_COMPOSITE_TABLE_NAME,
             partition_key=self.DYNAMODB_PARTITION_KEY,
             value_attribute_key=self.VALUE_ATTRIBUTE_KEY,
             partition_key_value=self.PARTITION_KEY_VALUE,
-            sort_key="sortKey",
-            sort_key_value="foo",
+            sort_key=self.DYNAMODB_SORT_KEY,
+            sort_key_value=self.SORT_KEY_VALUE,
         )
         memory = ConversationMemory(conversation_memory_driver=memory_driver)
         pipeline = Pipeline(conversation_memory=memory)
 
         pipeline.add_task(PromptTask("test"))
 
-        response = table.get_item(TableName=self.DYNAMODB_TABLE_NAME, Key={"entryId": "bar", "sortKey": "foo"})
+        response = table.get_item(
+            TableName=self.DYNAMODB_COMPOSITE_TABLE_NAME, Key={"entryId": "bar", "sortKey": "baz"}
+        )
         assert "Item" not in response
 
         pipeline.run()
 
-        response = table.get_item(TableName=self.DYNAMODB_TABLE_NAME, Key={"entryId": "bar", "sortKey": "foo"})
+        response = table.get_item(
+            TableName=self.DYNAMODB_COMPOSITE_TABLE_NAME, Key={"entryId": "bar", "sortKey": "baz"}
+        )
         assert "Item" in response
 
     def test_load(self):
@@ -109,12 +130,12 @@ def test_load(self):
     def test_load_with_sort_key(self):
         memory_driver = AmazonDynamoDbConversationMemoryDriver(
             session=boto3.Session(region_name=self.AWS_REGION),
-            table_name=self.DYNAMODB_TABLE_NAME,
+            table_name=self.DYNAMODB_COMPOSITE_TABLE_NAME,
             partition_key=self.DYNAMODB_PARTITION_KEY,
             value_attribute_key=self.VALUE_ATTRIBUTE_KEY,
             partition_key_value=self.PARTITION_KEY_VALUE,
-            sort_key="sortKey",
-            sort_key_value="foo",
+            sort_key=self.DYNAMODB_SORT_KEY,
+            sort_key_value=self.SORT_KEY_VALUE,
         )
         memory = ConversationMemory(conversation_memory_driver=memory_driver, meta={"foo": "bar"})
         pipeline = Pipeline(conversation_memory=memory)

tests/unit/drivers/prompt/test_anthropic_prompt_driver.py

@@ -14,8 +14,6 @@ class TestAnthropicPromptDriver:
         {
             "description": "test description: foo",
             "input_schema": {
-                "$id": "Input Schema",
-                "$schema": "http://json-schema.org/draft-07/schema#",
                 "additionalProperties": False,
                 "properties": {
                     "values": {
@@ -34,8 +32,6 @@ class TestAnthropicPromptDriver:
         {
             "description": "test description: foo",
             "input_schema": {
-                "$id": "Input Schema",
-                "$schema": "http://json-schema.org/draft-07/schema#",
                 "additionalProperties": False,
                 "properties": {
                     "values": {
@@ -54,8 +50,6 @@ class TestAnthropicPromptDriver:
         {
             "description": "test description: foo",
             "input_schema": {
-                "$id": "Input Schema",
-                "$schema": "http://json-schema.org/draft-07/schema#",
                 "additionalProperties": False,
                 "properties": {
                     "values": {
@@ -74,8 +68,6 @@ class TestAnthropicPromptDriver:
         {
             "description": "test description",
             "input_schema": {
-                "$id": "Input Schema",
-                "$schema": "http://json-schema.org/draft-07/schema#",
                 "additionalProperties": False,
                 "properties": {},
                 "required": [],
@@ -86,8 +78,6 @@ class TestAnthropicPromptDriver:
         {
             "description": "test description",
             "input_schema": {
-                "$id": "Input Schema",
-                "$schema": "http://json-schema.org/draft-07/schema#",
                 "additionalProperties": False,
                 "properties": {},
                 "required": [],
@@ -98,8 +88,6 @@ class TestAnthropicPromptDriver:
         {
             "description": "test description: foo",
             "input_schema": {
-                "$id": "Input Schema",
-                "$schema": "http://json-schema.org/draft-07/schema#",
                 "additionalProperties": False,
                 "properties": {
                     "values": {
@@ -118,8 +106,6 @@ class TestAnthropicPromptDriver:
         {
             "description": "test description",
             "input_schema": {
-                "$id": "Input Schema",
-                "$schema": "http://json-schema.org/draft-07/schema#",
                 "additionalProperties": False,
                 "properties": {
                     "values": {

tests/unit/loaders/test_pdf_loader.py

@@ -20,7 +20,7 @@ def test_load(self, loader, create_source):
 
         artifacts = loader.load(source)
 
-        assert len(artifacts) == 151
+        assert len(artifacts) == 156
         assert artifacts[0].value.startswith("Bitcoin: A Peer-to-Peer")
         assert artifacts[-1].value.endswith('its applications," 1957.\n9')
         assert artifacts[0].embedding == [0, 1]
@@ -37,7 +37,7 @@ def test_load_collection(self, loader, create_source):
 
         for key in keys:
             artifact = collection[key]
-            assert len(artifact) == 151
+            assert len(artifact) == 156
             assert artifact[0].value.startswith("Bitcoin: A Peer-to-Peer")
             assert artifact[-1].value.endswith('its applications," 1957.\n9')
             assert artifact[0].embedding == [0, 1]