diff --git a/.github/workflows/pr-Integration-tests.yml b/.github/workflows/pr-integration-tests.yml similarity index 95% rename from .github/workflows/pr-Integration-tests.yml rename to .github/workflows/pr-integration-tests.yml index a51dc743f33..f2dc97e75da 100644 --- a/.github/workflows/pr-Integration-tests.yml +++ b/.github/workflows/pr-integration-tests.yml @@ -13,7 +13,10 @@ on: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - + CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} + CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} + CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} + jobs: integration-tests: # See https://runs-on.com/runners/linux/ @@ -195,6 +198,9 @@ jobs: -e API_SERVER_HOST=api_server \ -e OPENAI_API_KEY=${OPENAI_API_KEY} \ -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \ + -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \ + -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \ + -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \ -e TEST_WEB_HOSTNAME=test-runner \ danswer/danswer-integration:test \ /app/tests/integration/tests \ diff --git a/backend/tests/integration/tests/connector/test_connector_creation.py b/backend/tests/integration/tests/connector/test_connector_creation.py new file mode 100644 index 00000000000..acfafe9436d --- /dev/null +++ b/backend/tests/integration/tests/connector/test_connector_creation.py @@ -0,0 +1,84 @@ +import os +from datetime import datetime +from datetime import timezone + +from danswer.server.documents.models import DocumentSource +from tests.integration.common_utils.managers.cc_pair import CCPairManager +from tests.integration.common_utils.managers.user import UserManager +from tests.integration.common_utils.test_models import DATestUser + + +def test_connector_creation(reset: None) -> None: + # Creating an admin user (first user created is automatically an admin) + admin_user: DATestUser = UserManager.create(name="admin_user") + + # create connectors + cc_pair_1 = CCPairManager.create_from_scratch( + source=DocumentSource.INGESTION_API, + user_performing_action=admin_user, + ) + + cc_pair_info = CCPairManager.get_single( + cc_pair_1.id, user_performing_action=admin_user + ) + assert cc_pair_info + assert cc_pair_info.creator + assert str(cc_pair_info.creator) == admin_user.id + assert cc_pair_info.creator_email == admin_user.email + + +def test_overlapping_connector_creation(reset: None) -> None: + """Tests that connectors indexing the same documents don't interfere with each other. + A previous bug involved document by cc pair entries not being added for new connectors + when the docs existed already via another connector and were up to date relative to the source. + """ + admin_user: DATestUser = UserManager.create(name="admin_user") + + config = { + "wiki_base": os.environ["CONFLUENCE_TEST_SPACE_URL"], + "space": "DailyConne", + "is_cloud": True, + "page_id": "", + } + + credential = { + "confluence_username": os.environ["CONFLUENCE_USER_NAME"], + "confluence_access_token": os.environ["CONFLUENCE_ACCESS_TOKEN"], + } + + # store the time before we create the connector so that we know after + # when the indexing should have started + now = datetime.now(timezone.utc) + + # create connector + cc_pair_1 = CCPairManager.create_from_scratch( + source=DocumentSource.CONFLUENCE, + connector_specific_config=config, + credential_json=credential, + user_performing_action=admin_user, + ) + + CCPairManager.wait_for_indexing( + cc_pair_1, now, timeout=120, user_performing_action=admin_user + ) + + now = datetime.now(timezone.utc) + + cc_pair_2 = CCPairManager.create_from_scratch( + source=DocumentSource.CONFLUENCE, + connector_specific_config=config, + credential_json=credential, + user_performing_action=admin_user, + ) + + CCPairManager.wait_for_indexing( + cc_pair_2, now, timeout=120, user_performing_action=admin_user + ) + + info_1 = CCPairManager.get_single(cc_pair_1.id, user_performing_action=admin_user) + assert info_1 + + info_2 = CCPairManager.get_single(cc_pair_2.id, user_performing_action=admin_user) + assert info_2 + + assert info_1.num_docs_indexed == info_2.num_docs_indexed diff --git a/backend/tests/integration/tests/connector/test_connector_deletion.py b/backend/tests/integration/tests/connector/test_connector_deletion.py index b14a75e0045..676ee4d9f4b 100644 --- a/backend/tests/integration/tests/connector/test_connector_deletion.py +++ b/backend/tests/integration/tests/connector/test_connector_deletion.py @@ -29,78 +29,6 @@ from tests.integration.common_utils.vespa import vespa_fixture -def test_connector_creation(reset: None) -> None: - # Creating an admin user (first user created is automatically an admin) - admin_user: DATestUser = UserManager.create(name="admin_user") - - # create connectors - cc_pair_1 = CCPairManager.create_from_scratch( - source=DocumentSource.INGESTION_API, - user_performing_action=admin_user, - ) - - cc_pair_info = CCPairManager.get_single( - cc_pair_1.id, user_performing_action=admin_user - ) - assert cc_pair_info - assert cc_pair_info.creator - assert str(cc_pair_info.creator) == admin_user.id - assert cc_pair_info.creator_email == admin_user.email - - -# TODO(rkuo): will enable this once i have credentials on github -# def test_overlapping_connector_creation(reset: None) -> None: -# # Creating an admin user (first user created is automatically an admin) -# admin_user: DATestUser = UserManager.create(name="admin_user") - -# config = { -# "wiki_base": os.environ["CONFLUENCE_TEST_SPACE_URL"], -# "space": os.environ["CONFLUENCE_TEST_SPACE"], -# "is_cloud": True, -# "page_id": "", -# } - -# credential = { -# "confluence_username": os.environ["CONFLUENCE_USER_NAME"], -# "confluence_access_token": os.environ["CONFLUENCE_ACCESS_TOKEN"], -# } - -# # store the time before we create the connector so that we know after -# # when the indexing should have started -# now = datetime.now(timezone.utc) - -# # create connector -# cc_pair_1 = CCPairManager.create_from_scratch( -# source=DocumentSource.CONFLUENCE, -# connector_specific_config=config, -# credential_json=credential, -# user_performing_action=admin_user, -# ) - -# CCPairManager.wait_for_indexing( -# cc_pair_1, now, timeout=60, user_performing_action=admin_user -# ) - -# cc_pair_2 = CCPairManager.create_from_scratch( -# source=DocumentSource.CONFLUENCE, -# connector_specific_config=config, -# credential_json=credential, -# user_performing_action=admin_user, -# ) - -# CCPairManager.wait_for_indexing( -# cc_pair_2, now, timeout=60, user_performing_action=admin_user -# ) - -# info_1 = CCPairManager.get_single(cc_pair_1.id) -# assert info_1 - -# info_2 = CCPairManager.get_single(cc_pair_2.id) -# assert info_2 - -# assert info_1.num_docs_indexed == info_2.num_docs_indexed - - def test_connector_deletion(reset: None, vespa_client: vespa_fixture) -> None: # Creating an admin user (first user created is automatically an admin) admin_user: DATestUser = UserManager.create(name="admin_user")