Skip to content

Commit

Permalink
Running rag e2e test with kubernetes docs.
Browse files Browse the repository at this point in the history 
    - Updating test_rag.py so the test can validate answers from the kubernetes documentation.
    - Updating cloudbuild.yaml to ingest the database with the kubernetes documentation.
  • Loading branch information
@german-grandas
german-grandas committed Sep 18, 2024
1 parent 9b05372 commit 015d3ff
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 158 deletions.
271 changes: 118 additions & 153 deletions applications/rag/tests/test_rag.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,160 +3,125 @@
import requests

def test_prompts(prompt_url):
testcases = [
{
"prompt": "List the cast of Squid Game",
"expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..",
"expected_substrings": ["Lee Jung-jae", "Park Hae-soo", "Wi Ha-jun", "Oh Young-soo", "Jung Ho-yeon", "Heo Sung-tae", "Kim Joo-ryoung", "Tripathi Anupam", "You Seong-joo", "Lee You-mi"],
},
{
"prompt": "When was Squid Game released?",
"expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..",
"expected_substrings": ["September 17, 2021"],
},
{
"prompt": "What is the rating of Squid Game?",
"expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..",
"expected_substrings": ["TV-MA"],
},
{
"prompt": "List the cast of Avatar: The Last Airbender",
"expected_context": "This is a TV Show in United States called Avatar: The Last Airbender added at May 15, 2020 whose director is and with cast: Zach Tyler, Mae Whitman, Jack De Sena, Dee Bradley Baker, Dante Basco, Jessie Flower, Mako Iwamatsu released at 2007. Its rating is: TV-Y7. Its duration is 3 Seasons. Its description is Siblings Katara and Sokka wake young Aang from a long hibernation and learn he's an Avatar, whose air-bending powers can defeat the evil Fire Nation..",
"expected_substrings": ["Zach Tyler", "Mae Whitman", "Jack De Sena", "Dee Bradley Baker", "Dante Basco", "Jessie Flower", "Mako Iwamatsu"],
},
{
"prompt": "When was Avatar: The Last Airbender added on Netflix?",
"expected_context": "This is a TV Show in United States called Avatar: The Last Airbender added at May 15, 2020 whose director is and with cast: Zach Tyler, Mae Whitman, Jack De Sena, Dee Bradley Baker, Dante Basco, Jessie Flower, Mako Iwamatsu released at 2007. Its rating is: TV-Y7. Its duration is 3 Seasons. Its description is Siblings Katara and Sokka wake young Aang from a long hibernation and learn he's an Avatar, whose air-bending powers can defeat the evil Fire Nation..",
"expected_substrings": ["May 15, 2020"],
},
{
"prompt": "What is the rating of Avatar: The Last Airbender?",
"expected_context": "This is a TV Show in United States called Avatar: The Last Airbender added at May 15, 2020 whose director is and with cast: Zach Tyler, Mae Whitman, Jack De Sena, Dee Bradley Baker, Dante Basco, Jessie Flower, Mako Iwamatsu released at 2007. Its rating is: TV-Y7. Its duration is 3 Seasons. Its description is Siblings Katara and Sokka wake young Aang from a long hibernation and learn he's an Avatar, whose air-bending powers can defeat the evil Fire Nation..",
"expected_substrings": ["TV-Y7"],
},
]

for testcase in testcases:
prompt = testcase["prompt"]
expected_context = testcase["expected_context"]
expected_substrings = testcase["expected_substrings"]

print(f"Testing prompt: {prompt}")
data = {"prompt": prompt}
json_payload = json.dumps(data)

headers = {'Content-Type': 'application/json'}
response = requests.post(prompt_url, data=json_payload, headers=headers)
response.raise_for_status()

response = response.json()
context = response['response']['context']
text = response['response']['text']
user_prompt = response['response']['user_prompt']

print(f"Reply: {text}")

assert user_prompt == prompt, f"unexpected user prompt: {user_prompt} != {prompt}"
assert context == expected_context, f"unexpected context: {context} != {expected_context}"

for substring in expected_substrings:
assert substring in text, f"substring {substring} not in response:\n {text}"
try:
testcases = [
{
"prompt": "What's kubernetes?",
},
{
"prompt": "How create a kubernetes cluster?",
},
{
"prompt": "What's kubectl?",
}
]

for testcase in testcases:
prompt = testcase["prompt"]

print(f"Testing prompt: {prompt}")
data = {"prompt": prompt}
json_payload = json.dumps(data)

headers = {'Content-Type': 'application/json'}
response = requests.post(prompt_url, data=json_payload, headers=headers)
response.raise_for_status()

response = response.json()
print(response)
text = response['response'].get('text')

print(f"Reply: {text}")

assert response != None, f"Not response found: {response}"
assert text != None, f"Not text"
except Exception as err:
print(err)
raise err

def test_prompts_nlp(prompt_url):
testcases = [
{
"prompt": "List the cast of Squid Game",
"nlpFilterLevel": "0",
"expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..",
"expected_substrings": ["Lee Jung-jae", "Park Hae-soo", "Wi Ha-jun", "Oh Young-soo", "Jung Ho-yeon", "Heo Sung-tae", "Kim Joo-ryoung", "Tripathi Anupam", "You Seong-joo", "Lee You-mi"],
},
{
"prompt": "Which movie has a building that blows up?",
"nlpFilterLevel": "0",
"expected_context": "This is a Movie in United States called Oklahoma City added at April 19, 2017 whose director is Barak Goodman and with cast: released at 2017. Its rating is: TV-14. Its duration is 102 min. Its description is Domestic terrorist Timothy McVeigh's bomb attack on an Oklahoma City federal building in 1995 is chronicled in this examination of the shocking event..",
"expected_substrings": ["The movie \"Oklahoma City\" released in 2017, directed by Barak Goodman, and starring a cast, depicts the bombing of an Oklahoma City federal building in 1995. The building that blows up in the movie is the Alfred P. Murrah Federal Building."],
},
{
"prompt": "Which movie has a building that blows up?",
"nlpFilterLevel": "50",
"expected_context": "This is a Movie in United States called Oklahoma City added at April 19, 2017 whose director is Barak Goodman and with cast: released at 2017. Its rating is: TV-14. Its duration is 102 min. Its description is Domestic terrorist Timothy McVeigh's bomb attack on an Oklahoma City federal building in 1995 is chronicled in this examination of the shocking event..",
"expected_substrings": ["The response is deemed inappropriate for display."],
},
{
"prompt": "List the cast of Squid Game",
"nlpFilterLevel": "100",
"expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..",
"expected_substrings": ["The response is deemed inappropriate for display."],
}
]

for testcase in testcases:
prompt = testcase["prompt"]
nlpFilterLevel = testcase["nlpFilterLevel"]
expected_context = testcase["expected_context"]
expected_substrings = testcase["expected_substrings"]

print(f"Testing prompt: {prompt}")
data = {"prompt": prompt, "nlpFilterLevel": nlpFilterLevel}
json_payload = json.dumps(data)

headers = {'Content-Type': 'application/json'}
response = requests.post(prompt_url, data=json_payload, headers=headers)
response.raise_for_status()

response = response.json()
context = response['response']['context']
text = response['response']['text']
user_prompt = response['response']['user_prompt']

print(f"Reply: {text}")

assert user_prompt == prompt, f"unexpected user prompt: {user_prompt} != {prompt}"
assert context == expected_context, f"unexpected context: {context} != {expected_context}"

for substring in expected_substrings:
assert substring in text, f"substring {substring} not in response:\n {text}"
try:
testcases = [
{
"prompt": "What's kubernetes?",
"nlpFilterLevel": "0",
},
{
"prompt": "What's kubernetes?",
"nlpFilterLevel": "100",
},
{
"prompt": "How create a kubernetes cluster?",
"nlpFilterLevel": "0",
},
{
"prompt": "What's kubectl?",
"nlpFilterLevel": "50",
}
]

for testcase in testcases:
prompt = testcase["prompt"]
nlpFilterLevel = testcase["nlpFilterLevel"]

print(f"Testing prompt: {prompt}")
data = {"prompt": prompt, "nlpFilterLevel": nlpFilterLevel}
json_payload = json.dumps(data)

headers = {'Content-Type': 'application/json'}
response = requests.post(prompt_url, data=json_payload, headers=headers)
response.raise_for_status()

response = response.json()

text = response['response']['text']


print(f"Reply: {text}")

assert response != None, f"Not response found: {response}"
assert text != None, f"Not text"
except Exception as err:
print(err)
raise err

def test_prompts_dlp(prompt_url):
testcases = [
{
"prompt": "who worked with Robert De Niro and name one film they collaborated?",
"inspectTemplate": "projects/gke-ai-eco-dev/locations/global/inspectTemplates/DO-NOT-DELETE-e2e-test-inspect-template",
"deidentifyTemplate": "projects/gke-ai-eco-dev/locations/global/deidentifyTemplates/DO-NOT-DELETE-e2e-test-de-identify-template",
"expected_context": "This is a Movie in United States called GoodFellas added at January 1, 2021 whose director is Martin Scorsese and with cast: Robert De Niro, Ray Liotta, Joe Pesci, Lorraine Bracco, Paul Sorvino, Frank Sivero, Tony Darrow, Mike Starr, Frank Vincent, Chuck Low released at 1990. Its rating is: R. Its duration is 145 min. Its description is Former mobster Henry Hill recounts his colorful yet violent rise and fall in a New York crime family – a high-rolling dream turned paranoid nightmare..",
"expected_substrings": ["[PERSON_NAME] has worked with many talented actors and directors throughout his career. One film he collaborated with [PERSON_NAME] is \"GoodFellas,\" which was released in 1990. In this movie, [PERSON_NAME] played the role of [PERSON_NAME], a former mobster who recounts his rise and fall in a New York crime family."],
},
]

for testcase in testcases:
prompt = testcase["prompt"]
inspectTemplate = testcase["inspectTemplate"]
deidentifyTemplate = testcase["deidentifyTemplate"]
expected_context = testcase["expected_context"]
expected_substrings = testcase["expected_substrings"]

print(f"Testing prompt: {prompt}")
data = {"prompt": prompt, "inspectTemplate": inspectTemplate, "deidentifyTemplate": deidentifyTemplate}
json_payload = json.dumps(data)

headers = {'Content-Type': 'application/json'}
response = requests.post(prompt_url, data=json_payload, headers=headers)
response.raise_for_status()

response = response.json()
context = response['response']['context']
text = response['response']['text']
user_prompt = response['response']['user_prompt']

print(f"Reply: {text}")

assert user_prompt == prompt, f"unexpected user prompt: {user_prompt} != {prompt}"
assert context == expected_context, f"unexpected context: {context} != {expected_context}"

for substring in expected_substrings:
assert substring in text, f"substring {substring} not in response:\n {text}"

prompt_url = sys.argv[1]
test_prompts(prompt_url)
test_prompts_nlp(prompt_url)
test_prompts_dlp(prompt_url)
try:
testcases = [
{
"prompt": "What's kubernetes?",
"inspectTemplate": "projects/globant-gke-ai-resources/locations/us-central1/inspectTemplates/gke-rag-application-inspect-template", #"projects/gke-ai-eco-dev/locations/global/inspectTemplates/DO-NOT-DELETE-e2e-test-inspect-template",
"deidentifyTemplate": "projects/globant-gke-ai-resources/locations/us-central1/deidentifyTemplates/gke-rag-application-deidentify-template" #"projects/gke-ai-eco-dev/locations/global/deidentifyTemplates/DO-NOT-DELETE-e2e-test-de-identify-template",
},
]

for testcase in testcases:
prompt = testcase["prompt"]
inspectTemplate = testcase["inspectTemplate"]
deidentifyTemplate = testcase["deidentifyTemplate"]

print(f"Testing prompt: {prompt}")
data = {"prompt": prompt, "inspectTemplate": inspectTemplate, "deidentifyTemplate": deidentifyTemplate}
json_payload = json.dumps(data)

headers = {'Content-Type': 'application/json'}
response = requests.post(prompt_url, data=json_payload, headers=headers)
response.raise_for_status()

response = response.json()
text = response['response']['text']


print(f"Reply: {text}")

assert response != None, f"Not response found: {response}"
assert text != None, f"Not text"
except Exception as err:
print(err)
raise err

if __name__ == "__main__":
prompt_url = sys.argv[1]
test_prompts(prompt_url)
test_prompts_nlp(prompt_url)
test_prompts_dlp(prompt_url)
10 changes: 5 additions & 5 deletions cloudbuild.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,15 +259,15 @@ steps:
echo "pass" > /workspace/rag_frontend_result.txt
cd /workspace/
sed -i "s/<username>/$$KAGGLE_USERNAME/g" ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb
sed -i "s/<token>/$$KAGGLE_KEY/g" ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb
gsutil cp ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb gs://gke-aieco-rag-$SHORT_SHA-$_BUILD_ID/
sed -i "s/<username>/$$KAGGLE_USERNAME/g" ./applications/rag/example_notebooks/rag-data-ingest-with-kubernetes-docs.ipynb
sed -i "s/<token>/$$KAGGLE_KEY/g" ./applications/rag/example_notebooks/rag-data-ingest-with-kubernetes-docs.ipynb
gsutil cp ./applications/rag/example_notebooks/rag-data-ingest-with-kubernetes-docs.ipynb gs://gke-aieco-rag-$SHORT_SHA-$_BUILD_ID/
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID $(kubectl get pod -l app=jupyterhub,component=hub -n rag-$SHORT_SHA-$_BUILD_ID -o jsonpath="{.items[0].metadata.name}") -- jupyterhub token admin --log-level=CRITICAL | xargs python3 ./applications/rag/notebook_starter.py
# Wait for jupyterhub to trigger notebook pod startup
sleep 5s
kubectl wait --for=condition=Ready pod/jupyter-admin -n rag-$SHORT_SHA-$_BUILD_ID --timeout=500s
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- jupyter nbconvert --to script /data/rag-kaggle-ray-sql-interactive.ipynb
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- ipython /data/rag-kaggle-ray-sql-interactive.py
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- jupyter nbconvert --to script /data/rag-data-ingest-with-kubernetes-docs.ipynb
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- ipython /data/rag-data-ingest-with-kubernetes-docs.py
python3 ./applications/rag/tests/test_rag.py "http://127.0.0.1:8081/prompt"
echo "pass" > /workspace/rag_prompt_result.txt
Expand Down

0 comments on commit 015d3ff

Please sign in to comment.