diff --git a/applications/rag/tests/test_rag.py b/applications/rag/tests/test_rag.py index d7da3a0e2..5ed2f4b3c 100644 --- a/applications/rag/tests/test_rag.py +++ b/applications/rag/tests/test_rag.py @@ -3,160 +3,125 @@ import requests def test_prompts(prompt_url): - testcases = [ - { - "prompt": "List the cast of Squid Game", - "expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..", - "expected_substrings": ["Lee Jung-jae", "Park Hae-soo", "Wi Ha-jun", "Oh Young-soo", "Jung Ho-yeon", "Heo Sung-tae", "Kim Joo-ryoung", "Tripathi Anupam", "You Seong-joo", "Lee You-mi"], - }, - { - "prompt": "When was Squid Game released?", - "expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..", - "expected_substrings": ["September 17, 2021"], - }, - { - "prompt": "What is the rating of Squid Game?", - "expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..", - "expected_substrings": ["TV-MA"], - }, - { - "prompt": "List the cast of Avatar: The Last Airbender", - "expected_context": "This is a TV Show in United States called Avatar: The Last Airbender added at May 15, 2020 whose director is and with cast: Zach Tyler, Mae Whitman, Jack De Sena, Dee Bradley Baker, Dante Basco, Jessie Flower, Mako Iwamatsu released at 2007. Its rating is: TV-Y7. Its duration is 3 Seasons. Its description is Siblings Katara and Sokka wake young Aang from a long hibernation and learn he's an Avatar, whose air-bending powers can defeat the evil Fire Nation..", - "expected_substrings": ["Zach Tyler", "Mae Whitman", "Jack De Sena", "Dee Bradley Baker", "Dante Basco", "Jessie Flower", "Mako Iwamatsu"], - }, - { - "prompt": "When was Avatar: The Last Airbender added on Netflix?", - "expected_context": "This is a TV Show in United States called Avatar: The Last Airbender added at May 15, 2020 whose director is and with cast: Zach Tyler, Mae Whitman, Jack De Sena, Dee Bradley Baker, Dante Basco, Jessie Flower, Mako Iwamatsu released at 2007. Its rating is: TV-Y7. Its duration is 3 Seasons. Its description is Siblings Katara and Sokka wake young Aang from a long hibernation and learn he's an Avatar, whose air-bending powers can defeat the evil Fire Nation..", - "expected_substrings": ["May 15, 2020"], - }, - { - "prompt": "What is the rating of Avatar: The Last Airbender?", - "expected_context": "This is a TV Show in United States called Avatar: The Last Airbender added at May 15, 2020 whose director is and with cast: Zach Tyler, Mae Whitman, Jack De Sena, Dee Bradley Baker, Dante Basco, Jessie Flower, Mako Iwamatsu released at 2007. Its rating is: TV-Y7. Its duration is 3 Seasons. Its description is Siblings Katara and Sokka wake young Aang from a long hibernation and learn he's an Avatar, whose air-bending powers can defeat the evil Fire Nation..", - "expected_substrings": ["TV-Y7"], - }, - ] - - for testcase in testcases: - prompt = testcase["prompt"] - expected_context = testcase["expected_context"] - expected_substrings = testcase["expected_substrings"] - - print(f"Testing prompt: {prompt}") - data = {"prompt": prompt} - json_payload = json.dumps(data) - - headers = {'Content-Type': 'application/json'} - response = requests.post(prompt_url, data=json_payload, headers=headers) - response.raise_for_status() - - response = response.json() - context = response['response']['context'] - text = response['response']['text'] - user_prompt = response['response']['user_prompt'] - - print(f"Reply: {text}") - - assert user_prompt == prompt, f"unexpected user prompt: {user_prompt} != {prompt}" - assert context == expected_context, f"unexpected context: {context} != {expected_context}" - - for substring in expected_substrings: - assert substring in text, f"substring {substring} not in response:\n {text}" + try: + testcases = [ + { + "prompt": "What's kubernetes?", + }, + { + "prompt": "How create a kubernetes cluster?", + }, + { + "prompt": "What's kubectl?", + } + ] + + for testcase in testcases: + prompt = testcase["prompt"] + + print(f"Testing prompt: {prompt}") + data = {"prompt": prompt} + json_payload = json.dumps(data) + + headers = {'Content-Type': 'application/json'} + response = requests.post(prompt_url, data=json_payload, headers=headers) + response.raise_for_status() + + response = response.json() + print(response) + text = response['response'].get('text') + + print(f"Reply: {text}") + + assert response != None, f"Not response found: {response}" + assert text != None, f"Not text" + except Exception as err: + print(err) + raise err def test_prompts_nlp(prompt_url): - testcases = [ - { - "prompt": "List the cast of Squid Game", - "nlpFilterLevel": "0", - "expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..", - "expected_substrings": ["Lee Jung-jae", "Park Hae-soo", "Wi Ha-jun", "Oh Young-soo", "Jung Ho-yeon", "Heo Sung-tae", "Kim Joo-ryoung", "Tripathi Anupam", "You Seong-joo", "Lee You-mi"], - }, - { - "prompt": "Which movie has a building that blows up?", - "nlpFilterLevel": "0", - "expected_context": "This is a Movie in United States called Oklahoma City added at April 19, 2017 whose director is Barak Goodman and with cast: released at 2017. Its rating is: TV-14. Its duration is 102 min. Its description is Domestic terrorist Timothy McVeigh's bomb attack on an Oklahoma City federal building in 1995 is chronicled in this examination of the shocking event..", - "expected_substrings": ["The movie \"Oklahoma City\" released in 2017, directed by Barak Goodman, and starring a cast, depicts the bombing of an Oklahoma City federal building in 1995. The building that blows up in the movie is the Alfred P. Murrah Federal Building."], - }, - { - "prompt": "Which movie has a building that blows up?", - "nlpFilterLevel": "50", - "expected_context": "This is a Movie in United States called Oklahoma City added at April 19, 2017 whose director is Barak Goodman and with cast: released at 2017. Its rating is: TV-14. Its duration is 102 min. Its description is Domestic terrorist Timothy McVeigh's bomb attack on an Oklahoma City federal building in 1995 is chronicled in this examination of the shocking event..", - "expected_substrings": ["The response is deemed inappropriate for display."], - }, - { - "prompt": "List the cast of Squid Game", - "nlpFilterLevel": "100", - "expected_context": "This is a TV Show in called Squid Game added at September 17, 2021 whose director is and with cast: Lee Jung-jae, Park Hae-soo, Wi Ha-jun, Oh Young-soo, Jung Ho-yeon, Heo Sung-tae, Kim Joo-ryoung, Tripathi Anupam, You Seong-joo, Lee You-mi released at 2021. Its rating is: TV-MA. Its duration is 1 Season. Its description is Hundreds of cash-strapped players accept a strange invitation to compete in children's games. Inside, a tempting prize awaits — with deadly high stakes..", - "expected_substrings": ["The response is deemed inappropriate for display."], - } - ] - - for testcase in testcases: - prompt = testcase["prompt"] - nlpFilterLevel = testcase["nlpFilterLevel"] - expected_context = testcase["expected_context"] - expected_substrings = testcase["expected_substrings"] - - print(f"Testing prompt: {prompt}") - data = {"prompt": prompt, "nlpFilterLevel": nlpFilterLevel} - json_payload = json.dumps(data) - - headers = {'Content-Type': 'application/json'} - response = requests.post(prompt_url, data=json_payload, headers=headers) - response.raise_for_status() - - response = response.json() - context = response['response']['context'] - text = response['response']['text'] - user_prompt = response['response']['user_prompt'] - - print(f"Reply: {text}") - - assert user_prompt == prompt, f"unexpected user prompt: {user_prompt} != {prompt}" - assert context == expected_context, f"unexpected context: {context} != {expected_context}" - - for substring in expected_substrings: - assert substring in text, f"substring {substring} not in response:\n {text}" + try: + testcases = [ + { + "prompt": "What's kubernetes?", + "nlpFilterLevel": "0", + }, + { + "prompt": "What's kubernetes?", + "nlpFilterLevel": "100", + }, + { + "prompt": "How create a kubernetes cluster?", + "nlpFilterLevel": "0", + }, + { + "prompt": "What's kubectl?", + "nlpFilterLevel": "50", + } + ] + + for testcase in testcases: + prompt = testcase["prompt"] + nlpFilterLevel = testcase["nlpFilterLevel"] + + print(f"Testing prompt: {prompt}") + data = {"prompt": prompt, "nlpFilterLevel": nlpFilterLevel} + json_payload = json.dumps(data) + + headers = {'Content-Type': 'application/json'} + response = requests.post(prompt_url, data=json_payload, headers=headers) + response.raise_for_status() + + response = response.json() + + text = response['response']['text'] + + + print(f"Reply: {text}") + + assert response != None, f"Not response found: {response}" + assert text != None, f"Not text" + except Exception as err: + print(err) + raise err def test_prompts_dlp(prompt_url): - testcases = [ - { - "prompt": "who worked with Robert De Niro and name one film they collaborated?", - "inspectTemplate": "projects/gke-ai-eco-dev/locations/global/inspectTemplates/DO-NOT-DELETE-e2e-test-inspect-template", - "deidentifyTemplate": "projects/gke-ai-eco-dev/locations/global/deidentifyTemplates/DO-NOT-DELETE-e2e-test-de-identify-template", - "expected_context": "This is a Movie in United States called GoodFellas added at January 1, 2021 whose director is Martin Scorsese and with cast: Robert De Niro, Ray Liotta, Joe Pesci, Lorraine Bracco, Paul Sorvino, Frank Sivero, Tony Darrow, Mike Starr, Frank Vincent, Chuck Low released at 1990. Its rating is: R. Its duration is 145 min. Its description is Former mobster Henry Hill recounts his colorful yet violent rise and fall in a New York crime family – a high-rolling dream turned paranoid nightmare..", - "expected_substrings": ["[PERSON_NAME] has worked with many talented actors and directors throughout his career. One film he collaborated with [PERSON_NAME] is \"GoodFellas,\" which was released in 1990. In this movie, [PERSON_NAME] played the role of [PERSON_NAME], a former mobster who recounts his rise and fall in a New York crime family."], - }, - ] - - for testcase in testcases: - prompt = testcase["prompt"] - inspectTemplate = testcase["inspectTemplate"] - deidentifyTemplate = testcase["deidentifyTemplate"] - expected_context = testcase["expected_context"] - expected_substrings = testcase["expected_substrings"] - - print(f"Testing prompt: {prompt}") - data = {"prompt": prompt, "inspectTemplate": inspectTemplate, "deidentifyTemplate": deidentifyTemplate} - json_payload = json.dumps(data) - - headers = {'Content-Type': 'application/json'} - response = requests.post(prompt_url, data=json_payload, headers=headers) - response.raise_for_status() - - response = response.json() - context = response['response']['context'] - text = response['response']['text'] - user_prompt = response['response']['user_prompt'] - - print(f"Reply: {text}") - - assert user_prompt == prompt, f"unexpected user prompt: {user_prompt} != {prompt}" - assert context == expected_context, f"unexpected context: {context} != {expected_context}" - - for substring in expected_substrings: - assert substring in text, f"substring {substring} not in response:\n {text}" - -prompt_url = sys.argv[1] -test_prompts(prompt_url) -test_prompts_nlp(prompt_url) -test_prompts_dlp(prompt_url) + try: + testcases = [ + { + "prompt": "What's kubernetes?", + "inspectTemplate": "projects/globant-gke-ai-resources/locations/us-central1/inspectTemplates/gke-rag-application-inspect-template", #"projects/gke-ai-eco-dev/locations/global/inspectTemplates/DO-NOT-DELETE-e2e-test-inspect-template", + "deidentifyTemplate": "projects/globant-gke-ai-resources/locations/us-central1/deidentifyTemplates/gke-rag-application-deidentify-template" #"projects/gke-ai-eco-dev/locations/global/deidentifyTemplates/DO-NOT-DELETE-e2e-test-de-identify-template", + }, + ] + + for testcase in testcases: + prompt = testcase["prompt"] + inspectTemplate = testcase["inspectTemplate"] + deidentifyTemplate = testcase["deidentifyTemplate"] + + print(f"Testing prompt: {prompt}") + data = {"prompt": prompt, "inspectTemplate": inspectTemplate, "deidentifyTemplate": deidentifyTemplate} + json_payload = json.dumps(data) + + headers = {'Content-Type': 'application/json'} + response = requests.post(prompt_url, data=json_payload, headers=headers) + response.raise_for_status() + + response = response.json() + text = response['response']['text'] + + + print(f"Reply: {text}") + + assert response != None, f"Not response found: {response}" + assert text != None, f"Not text" + except Exception as err: + print(err) + raise err + +if __name__ == "__main__": + prompt_url = sys.argv[1] + test_prompts(prompt_url) + test_prompts_nlp(prompt_url) + test_prompts_dlp(prompt_url) \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml index d3c22f362..59d99b7a0 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -259,15 +259,15 @@ steps: echo "pass" > /workspace/rag_frontend_result.txt cd /workspace/ - sed -i "s//$$KAGGLE_USERNAME/g" ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb - sed -i "s//$$KAGGLE_KEY/g" ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb - gsutil cp ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb gs://gke-aieco-rag-$SHORT_SHA-$_BUILD_ID/ + sed -i "s//$$KAGGLE_USERNAME/g" ./applications/rag/example_notebooks/rag-data-ingest-with-kubernetes-docs.ipynb + sed -i "s//$$KAGGLE_KEY/g" ./applications/rag/example_notebooks/rag-data-ingest-with-kubernetes-docs.ipynb + gsutil cp ./applications/rag/example_notebooks/rag-data-ingest-with-kubernetes-docs.ipynb gs://gke-aieco-rag-$SHORT_SHA-$_BUILD_ID/ kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID $(kubectl get pod -l app=jupyterhub,component=hub -n rag-$SHORT_SHA-$_BUILD_ID -o jsonpath="{.items[0].metadata.name}") -- jupyterhub token admin --log-level=CRITICAL | xargs python3 ./applications/rag/notebook_starter.py # Wait for jupyterhub to trigger notebook pod startup sleep 5s kubectl wait --for=condition=Ready pod/jupyter-admin -n rag-$SHORT_SHA-$_BUILD_ID --timeout=500s - kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- jupyter nbconvert --to script /data/rag-kaggle-ray-sql-interactive.ipynb - kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- ipython /data/rag-kaggle-ray-sql-interactive.py + kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- jupyter nbconvert --to script /data/rag-data-ingest-with-kubernetes-docs.ipynb + kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- ipython /data/rag-data-ingest-with-kubernetes-docs.py python3 ./applications/rag/tests/test_rag.py "http://127.0.0.1:8081/prompt" echo "pass" > /workspace/rag_prompt_result.txt