Implement backoff strategy

MANIFEST.in

@@ -0,0 +1 @@
+include requirements.txt requirements-dev.txt

docs/evaluators/index.md

@@ -55,6 +55,12 @@ The endpoint URL for the AWS service. If unspecified, the public endpoint based
 
 ---
 
+`max_retry` _(integer; optional)_
+
+The maximum number of retry attempts. The default is `10`.
+
+---
+
 ### Available Evaluators
 
 - [Anthropic Claude](./bedrock/claude.md)

docs/index.md

@@ -31,7 +31,7 @@ Agent Evaluation requires `python>=3.9`. Please make sure you have an acceptable
 To install:
 
 ```bash
-pip install agenteval
+pip install agent-evaluation
 ```
 
 You can also install from source by cloning the [repo](https://github.com/awslabs/agent-evaluation) and installing from the project root.

docs/stylesheets/extra.css

@@ -1,7 +1,7 @@
 html {
-  font-size: 20px;
+  font-size: 22px;
 }
 
 .md-typeset .admonition {
-  font-size: 14px;
+  font-size: 15px;
 }

docs/targets/index.md

@@ -13,6 +13,7 @@ target:
   aws_profile:
   aws_region:
   endpoint_url:
+  max_retry:
 ```
 
 `aws_profile` _(string; optional)_
@@ -33,6 +34,12 @@ The endpoint URL for the AWS service. If unspecified, the public endpoint based
 
 ---
 
+`max_retry` _(integer; optional)_
+
+The maximum number of retry attempts. The default is `10`.
+
+---
+
 ### Available Targets
 
 - [Agents for Amazon Bedrock](./aws/bedrock_agents.md)

setup.py

@@ -3,6 +3,7 @@
 
 from setuptools import find_packages
 
+DIST_NAME = "agent-evaluation"
 VERSION = "0.1.0"
 DESCRIPTION = "A LLM-powered framework for testing virtual agents."
 AUTHOR = "Amazon Web Services"
@@ -23,13 +24,14 @@ def read(fname):
 
 
 setup(
-    name="agenteval",
+    name=DIST_NAME,
     version=VERSION,
     packages=find_packages(where=PACKAGE_DIR),
     package_dir={"": PACKAGE_DIR},
     package_data=PACKAGE_DATA,
     description=DESCRIPTION,
     long_description=read("README.md"),
+    long_description_content_type="text/markdown",
     python_requires=REQUIRES_PYTHON,
     install_requires=read("requirements.txt").splitlines(),
     extras_require={"dev": read("requirements-dev.txt").splitlines()},

src/agenteval/__init__.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 __version__ = "0.1.0"
 
 import logging

src/agenteval/cli.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import logging
 import os
 from typing import Optional

src/agenteval/conversation_handler.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 _USER = "USER"
 _AGENT = "AGENT"
 _START_TURN_COUNT = 0

src/agenteval/defaults.py

@@ -1 +1,4 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 MAX_TURNS = 2

src/agenteval/evaluators/__init__.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from .evaluator import BaseEvaluator
 
 __all__ = ["BaseEvaluator"]

src/agenteval/evaluators/aws/__init__.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from .aws_evaluator import AWSEvaluator
 
 __all__ = ["AWSEvaluator"]

src/agenteval/evaluators/aws/aws_evaluator.py

@@ -1,9 +1,12 @@
-from typing import Optional
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
 
-import boto3
-from botocore.client import BaseClient
+from typing import Optional
 
 from agenteval.evaluators import BaseEvaluator
+from agenteval.utils import create_boto3_client
+
+_DEFAULT_MAX_RETRY = 10
 
 
 class AWSEvaluator(BaseEvaluator):
@@ -19,6 +22,7 @@ def __init__(
         aws_profile: Optional[str] = None,
         aws_region: Optional[str] = None,
         endpoint_url: Optional[str] = None,
+        max_retry: int = _DEFAULT_MAX_RETRY,
         **kwargs,
     ):
         """
@@ -29,34 +33,15 @@ def __init__(
             aws_profile (str, optional): The AWS profile name.
             aws_region (str, optional): The AWS region.
             endpoint_url (str, optional): The endpoint URL for the AWS service.
+            max_retry (int, optional): The maximum number of retry attempts.
             **kwargs : Arguments for the BaseEvaluator initializer.
         """
         super().__init__(**kwargs)
 
-        self.boto3_client = self._create_boto3_client(
+        self.boto3_client = create_boto3_client(
             boto3_service_name=boto3_service_name,
             aws_profile=aws_profile,
             aws_region=aws_region,
             endpoint_url=endpoint_url,
+            max_retry=max_retry,
         )
-
-    @staticmethod
-    def _create_boto3_client(
-        boto3_service_name: str,
-        aws_profile: Optional[str],
-        aws_region: Optional[str],
-        endpoint_url: Optional[str],
-    ) -> BaseClient:
-        """Create a `boto3` client.
-
-        Args:
-            boto3_service_name (str): The `boto3` service name (e.g `"bedrock-runtime"`).
-            aws_profile (str, optional): The AWS profile name.
-            aws_region (str, optional): The AWS region.
-            endpoint_url (str, optional): The endpoint URL for the AWS service.
-
-        Returns:
-            BaseClient
-        """
-        session = boto3.Session(profile_name=aws_profile, region_name=aws_region)
-        return session.client(boto3_service_name, endpoint_url=endpoint_url)

src/agenteval/evaluators/aws/bedrock/__init__.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from .bedrock_evaluator import BedrockEvaluator
 from .claude.claude_evaluator import ClaudeEvaluator
 

src/agenteval/evaluators/aws/bedrock/bedrock_evaluator.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import json
 
 from agenteval.evaluators.aws import AWSEvaluator

src/agenteval/evaluators/aws/bedrock/claude/__init__.py

@@ -0,0 +1,2 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0

src/agenteval/evaluators/aws/bedrock/claude/claude_evaluator.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import json
 import logging
 import os

src/agenteval/evaluators/aws/bedrock/claude/model_configs.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 CLAUDE_MODEL_ID_MAP = {
     "claude": "anthropic.claude-v2:1",
     "claude-sonnet": "anthropic.claude-3-sonnet-20240229-v1:0",

src/agenteval/evaluators/evaluator.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from abc import ABC, abstractmethod
 from typing import Optional
 

src/agenteval/hook.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from agenteval.test import Test
 from agenteval.test_result import TestResult
 from agenteval.trace_handler import TraceHandler

src/agenteval/plan.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 import logging

src/agenteval/runner/__init__.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from .runner import Runner
 
 __all__ = ["Runner"]

src/agenteval/runner/runner.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import concurrent.futures
 import logging
 import os

src/agenteval/runner/summary.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import logging
 import os
 

src/agenteval/targets/__init__.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from .target import BaseTarget, TargetResponse
 
 __all__ = ["TargetResponse", "BaseTarget"]

src/agenteval/targets/aws/aws_target.py

@@ -1,9 +1,9 @@
 from typing import Optional
 
-import boto3
-from botocore.client import BaseClient
-
 from agenteval.targets import BaseTarget
+from agenteval.utils import create_boto3_client
+
+_DEFAULT_MAX_RETRY = 10
 
 
 class AWSTarget(BaseTarget):
@@ -19,6 +19,7 @@ def __init__(
         aws_profile: Optional[str] = None,
         aws_region: Optional[str] = None,
         endpoint_url: Optional[str] = None,
+        max_retry: int = _DEFAULT_MAX_RETRY,
     ):
         """
         Initialize the AWS target.
@@ -28,31 +29,12 @@ def __init__(
             aws_profile (str, optional): The AWS profile name.
             aws_region (str, optional): The AWS region.
             endpoint_url (str, optional): The endpoint URL for the AWS service.
+            max_retry (int, optional): The maximum number of retry attempts.
         """
-        self.boto3_client = self._create_boto3_client(
+        self.boto3_client = create_boto3_client(
             boto3_service_name=boto3_service_name,
             aws_profile=aws_profile,
             aws_region=aws_region,
             endpoint_url=endpoint_url,
+            max_retry=max_retry,
         )
-
-    @staticmethod
-    def _create_boto3_client(
-        boto3_service_name: str,
-        aws_profile: Optional[str],
-        aws_region: Optional[str],
-        endpoint_url: Optional[str],
-    ) -> BaseClient:
-        """Create a `boto3` client.
-
-        Args:
-            boto3_service_name (str): The `boto3` service name (e.g `"bedrock-agent-runtime"`).
-            aws_profile (str, optional): The AWS profile name.
-            aws_region (str, optional): The AWS region.
-            endpoint_url (str, optional): The endpoint URL for the AWS service.
-
-        Returns:
-            BaseClient
-        """
-        session = boto3.Session(profile_name=aws_profile, region_name=aws_region)
-        return session.client(boto3_service_name, endpoint_url=endpoint_url)

src/agenteval/targets/target.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 from abc import ABC, abstractmethod

src/agenteval/test.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Optional
 
 from pydantic import BaseModel

src/agenteval/test_result.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from pydantic import BaseModel
 
 from agenteval.conversation_handler import ConversationHandler

src/agenteval/trace_handler.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import inspect
 import json
 import os

src/agenteval/utils/__init__.py

@@ -1,3 +1,7 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from .aws import create_boto3_client
 from .imports import import_class
 
-__all__ = ["import_class"]
+__all__ = ["import_class", "create_boto3_client"]

src/agenteval/utils/aws.py

@@ -0,0 +1,36 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Optional
+
+import boto3
+from botocore.client import BaseClient
+from botocore.config import Config
+
+_RETRY_MODE = "adaptive"
+
+
+def create_boto3_client(
+    boto3_service_name: str,
+    aws_profile: Optional[str],
+    aws_region: Optional[str],
+    endpoint_url: Optional[str],
+    max_retry: int,
+) -> BaseClient:
+    """Create a `boto3` client.
+
+    Args:
+        boto3_service_name (str): The `boto3` service name (e.g `"bedrock-runtime"`).
+        aws_profile (str, optional): The AWS profile name.
+        aws_region (str, optional): The AWS region.
+        endpoint_url (str, optional): The endpoint URL for the AWS service.
+        max_retry (int, optional): The maximum number of retry attempts.
+
+    Returns:
+        BaseClient
+    """
+
+    config = Config(retries={"max_attempts": max_retry, "mode": _RETRY_MODE})
+
+    session = boto3.Session(profile_name=aws_profile, region_name=aws_region)
+    return session.client(boto3_service_name, endpoint_url=endpoint_url, config=config)

src/agenteval/utils/imports.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 from importlib import import_module
 from typing import Optional