diff --git a/.github/actions/custom-build-and-push/action.yml b/.github/actions/custom-build-and-push/action.yml
index 48344237059..fbee0554995 100644
--- a/.github/actions/custom-build-and-push/action.yml
+++ b/.github/actions/custom-build-and-push/action.yml
@@ -32,16 +32,20 @@ inputs:
description: 'Cache destinations'
required: false
retry-wait-time:
- description: 'Time to wait before retry in seconds'
+ description: 'Time to wait before attempt 2 in seconds'
required: false
- default: '5'
+ default: '60'
+ retry-wait-time-2:
+ description: 'Time to wait before attempt 3 in seconds'
+ required: false
+ default: '120'
runs:
using: "composite"
steps:
- - name: Build and push Docker image (First Attempt)
+ - name: Build and push Docker image (Attempt 1 of 3)
id: buildx1
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
continue-on-error: true
with:
context: ${{ inputs.context }}
@@ -54,16 +58,39 @@ runs:
cache-from: ${{ inputs.cache-from }}
cache-to: ${{ inputs.cache-to }}
- - name: Wait to retry
+ - name: Wait before attempt 2
if: steps.buildx1.outcome != 'success'
run: |
echo "First attempt failed. Waiting ${{ inputs.retry-wait-time }} seconds before retry..."
sleep ${{ inputs.retry-wait-time }}
shell: bash
- - name: Build and push Docker image (Retry Attempt)
+ - name: Build and push Docker image (Attempt 2 of 3)
+ id: buildx2
if: steps.buildx1.outcome != 'success'
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
+ with:
+ context: ${{ inputs.context }}
+ file: ${{ inputs.file }}
+ platforms: ${{ inputs.platforms }}
+ pull: ${{ inputs.pull }}
+ push: ${{ inputs.push }}
+ load: ${{ inputs.load }}
+ tags: ${{ inputs.tags }}
+ cache-from: ${{ inputs.cache-from }}
+ cache-to: ${{ inputs.cache-to }}
+
+ - name: Wait before attempt 3
+ if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
+ run: |
+ echo "Second attempt failed. Waiting ${{ inputs.retry-wait-time-2 }} seconds before retry..."
+ sleep ${{ inputs.retry-wait-time-2 }}
+ shell: bash
+
+ - name: Build and push Docker image (Attempt 3 of 3)
+ id: buildx3
+ if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
+ uses: docker/build-push-action@v6
with:
context: ${{ inputs.context }}
file: ${{ inputs.file }}
@@ -74,3 +101,9 @@ runs:
tags: ${{ inputs.tags }}
cache-from: ${{ inputs.cache-from }}
cache-to: ${{ inputs.cache-to }}
+
+ - name: Report failure
+ if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success' && steps.buildx3.outcome != 'success'
+ run: |
+ echo "All attempts failed. Possible transient infrastucture issues? Try again later or inspect logs for details."
+ shell: bash
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index e57283f0377..8287f9b5300 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -6,20 +6,24 @@
[Describe the tests you ran to verify your changes]
-## Accepted Risk
-[Any know risks or failure modes to point out to reviewers]
+## Accepted Risk (provide if relevant)
+N/A
-## Related Issue(s)
-[If applicable, link to the issue(s) this PR addresses]
+## Related Issue(s) (provide if relevant)
+N/A
-## Checklist:
-- [ ] All of the automated tests pass
-- [ ] All PR comments are addressed and marked resolved
-- [ ] If there are migrations, they have been rebased to latest main
-- [ ] If there are new dependencies, they are added to the requirements
-- [ ] If there are new environment variables, they are added to all of the deployment methods
-- [ ] If there are new APIs that don't require auth, they are added to PUBLIC_ENDPOINT_SPECS
-- [ ] Docker images build and basic functionalities work
-- [ ] Author has done a final read through of the PR right before merge
+## Mental Checklist:
+- All of the automated tests pass
+- All PR comments are addressed and marked resolved
+- If there are migrations, they have been rebased to latest main
+- If there are new dependencies, they are added to the requirements
+- If there are new environment variables, they are added to all of the deployment methods
+- If there are new APIs that don't require auth, they are added to PUBLIC_ENDPOINT_SPECS
+- Docker images build and basic functionalities work
+- Author has done a final read through of the PR right before merge
+
+## Backporting (check the box to trigger backport action)
+Note: You have to check that the action passes, otherwise resolve the conflicts manually and tag the patches.
+- [ ] This PR should be backported (make sure to check that the backport attempt succeeds)
diff --git a/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml b/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml
new file mode 100644
index 00000000000..871c96841ad
--- /dev/null
+++ b/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml
@@ -0,0 +1,136 @@
+name: Build and Push Cloud Web Image on Tag
+# Identical to the web container build, but with correct image tag and build args
+
+on:
+ push:
+ tags:
+ - '*'
+
+env:
+ REGISTRY_IMAGE: danswer/danswer-cloud-web-server
+ LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
+
+jobs:
+ build:
+ runs-on:
+ - runs-on
+ - runner=${{ matrix.platform == 'linux/amd64' && '8cpu-linux-x64' || '8cpu-linux-arm64' }}
+ - run-id=${{ github.run_id }}
+ - tag=platform-${{ matrix.platform }}
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - linux/amd64
+ - linux/arm64
+
+ steps:
+ - name: Prepare
+ run: |
+ platform=${{ matrix.platform }}
+ echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY_IMAGE }}
+ tags: |
+ type=raw,value=${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+ type=raw,value=${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_TOKEN }}
+
+ - name: Build and push by digest
+ id: build
+ uses: docker/build-push-action@v5
+ with:
+ context: ./web
+ file: ./web/Dockerfile
+ platforms: ${{ matrix.platform }}
+ push: true
+ build-args: |
+ DANSWER_VERSION=${{ github.ref_name }}
+ NEXT_PUBLIC_CLOUD_ENABLED=true
+ NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }}
+ NEXT_PUBLIC_POSTHOG_HOST=${{ secrets.POSTHOG_HOST }}
+ NEXT_PUBLIC_SENTRY_DSN=${{ secrets.SENTRY_DSN }}
+ # needed due to weird interactions with the builds for different platforms
+ no-cache: true
+ labels: ${{ steps.meta.outputs.labels }}
+ outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+ - name: Export digest
+ run: |
+ mkdir -p /tmp/digests
+ digest="${{ steps.build.outputs.digest }}"
+ touch "/tmp/digests/${digest#sha256:}"
+
+ - name: Upload digest
+ uses: actions/upload-artifact@v4
+ with:
+ name: digests-${{ env.PLATFORM_PAIR }}
+ path: /tmp/digests/*
+ if-no-files-found: error
+ retention-days: 1
+
+ merge:
+ runs-on: ubuntu-latest
+ needs:
+ - build
+ steps:
+ - name: Download digests
+ uses: actions/download-artifact@v4
+ with:
+ path: /tmp/digests
+ pattern: digests-*
+ merge-multiple: true
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY_IMAGE }}
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_TOKEN }}
+
+ - name: Create manifest list and push
+ working-directory: /tmp/digests
+ run: |
+ docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+ $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+
+ - name: Inspect image
+ run: |
+ docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+
+ # trivy has their own rate limiting issues causing this action to flake
+ # we worked around it by hardcoding to different db repos in env
+ # can re-enable when they figure it out
+ # https://github.com/aquasecurity/trivy/discussions/7538
+ # https://github.com/aquasecurity/trivy-action/issues/389
+ - name: Run Trivy vulnerability scanner
+ uses: aquasecurity/trivy-action@master
+ env:
+ TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+ TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
+ with:
+ image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+ severity: 'CRITICAL,HIGH'
diff --git a/.github/workflows/helm-build-push.yml b/.github/workflows/helm-build-push.yml
index a9080239b40..a591eaecd3c 100644
--- a/.github/workflows/helm-build-push.yml
+++ b/.github/workflows/helm-build-push.yml
@@ -62,7 +62,7 @@ jobs:
permissions:
contents: write
runs-on: ubuntu-latest
- # needs: helm_chart_version_check
+ needs: helm_chart_version_check
# if: ${{ needs.helm_chart_version_check.outputs.version_changed == 'true' }}
steps:
- name: Checkout
diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
new file mode 100644
index 00000000000..0e921f8d694
--- /dev/null
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -0,0 +1,172 @@
+# This workflow is intended to be manually triggered via the GitHub Action tab.
+# Given a hotfix branch, it will attempt to open a PR to all release branches and
+# by default auto merge them
+
+name: Hotfix release branches
+
+on:
+ workflow_dispatch:
+ inputs:
+ hotfix_commit:
+ description: 'Hotfix commit hash'
+ required: true
+ hotfix_suffix:
+ description: 'Hotfix branch suffix (e.g. hotfix/v0.8-{suffix})'
+ required: true
+ release_branch_pattern:
+ description: 'Release branch pattern (regex)'
+ required: true
+ default: 'release/.*'
+ auto_merge:
+ description: 'Automatically merge the hotfix PRs'
+ required: true
+ type: choice
+ default: 'true'
+ options:
+ - true
+ - false
+
+jobs:
+ hotfix_release_branches:
+ permissions: write-all
+ # See https://runs-on.com/runners/linux/
+ # use a lower powered instance since this just does i/o to docker hub
+ runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+ steps:
+
+ # needs RKUO_DEPLOY_KEY for write access to merge PR's
+ - name: Checkout Repository
+ uses: actions/checkout@v4
+ with:
+ ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
+ fetch-depth: 0
+
+ - name: Set up Git user
+ run: |
+ git config user.name "Richard Kuo [bot]"
+ git config user.email "rkuo[bot]@danswer.ai"
+
+ - name: Fetch All Branches
+ run: |
+ git fetch --all --prune
+
+ - name: Verify Hotfix Commit Exists
+ run: |
+ git rev-parse --verify "${{ github.event.inputs.hotfix_commit }}" || { echo "Commit not found: ${{ github.event.inputs.hotfix_commit }}"; exit 1; }
+
+ - name: Get Release Branches
+ id: get_release_branches
+ run: |
+ BRANCHES=$(git branch -r | grep -E "${{ github.event.inputs.release_branch_pattern }}" | sed 's|origin/||' | tr -d ' ')
+ if [ -z "$BRANCHES" ]; then
+ echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'."
+ exit 1
+ fi
+
+ echo "Found release branches:"
+ echo "$BRANCHES"
+
+ # Join the branches into a single line separated by commas
+ BRANCHES_JOINED=$(echo "$BRANCHES" | tr '\n' ',' | sed 's/,$//')
+
+ # Set the branches as an output
+ echo "branches=$BRANCHES_JOINED" >> $GITHUB_OUTPUT
+
+ # notes on all the vagaries of wiring up automated PR's
+ # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs
+ # we must use a custom token for GH_TOKEN to trigger the subsequent PR checks
+ - name: Create and Merge Pull Requests to Matching Release Branches
+ env:
+ HOTFIX_COMMIT: ${{ github.event.inputs.hotfix_commit }}
+ HOTFIX_SUFFIX: ${{ github.event.inputs.hotfix_suffix }}
+ AUTO_MERGE: ${{ github.event.inputs.auto_merge }}
+ GH_TOKEN: ${{ secrets.RKUO_PERSONAL_ACCESS_TOKEN }}
+ run: |
+ # Get the branches from the previous step
+ BRANCHES="${{ steps.get_release_branches.outputs.branches }}"
+
+ # Convert BRANCHES to an array
+ IFS=$',' read -ra BRANCH_ARRAY <<< "$BRANCHES"
+
+ # Loop through each release branch and create and merge a PR
+ for RELEASE_BRANCH in "${BRANCH_ARRAY[@]}"; do
+ echo "Processing $RELEASE_BRANCH..."
+
+ # Parse out the release version by removing "release/" from the branch name
+ RELEASE_VERSION=${RELEASE_BRANCH#release/}
+ echo "Release version parsed: $RELEASE_VERSION"
+
+ HOTFIX_BRANCH="hotfix/${RELEASE_VERSION}-${HOTFIX_SUFFIX}"
+ echo "Creating PR from $HOTFIX_BRANCH to $RELEASE_BRANCH"
+
+ # Checkout the release branch
+ echo "Checking out $RELEASE_BRANCH"
+ git checkout "$RELEASE_BRANCH"
+
+ # Create the new hotfix branch
+ if git rev-parse --verify "$HOTFIX_BRANCH" >/dev/null 2>&1; then
+ echo "Hotfix branch $HOTFIX_BRANCH already exists. Skipping branch creation."
+ else
+ echo "Branching $RELEASE_BRANCH to $HOTFIX_BRANCH"
+ git checkout -b "$HOTFIX_BRANCH"
+ fi
+
+ # Check if the hotfix commit is a merge commit
+ if git rev-list --merges -n 1 "$HOTFIX_COMMIT" >/dev/null 2>&1; then
+ # -m 1 uses the target branch as the base (which is what we want)
+ echo "Hotfix commit $HOTFIX_COMMIT is a merge commit, using -m 1 for cherry-pick"
+ CHERRY_PICK_CMD="git cherry-pick -m 1 $HOTFIX_COMMIT"
+ else
+ CHERRY_PICK_CMD="git cherry-pick $HOTFIX_COMMIT"
+ fi
+
+ # Perform the cherry-pick
+ echo "Executing: $CHERRY_PICK_CMD"
+ eval "$CHERRY_PICK_CMD"
+
+ if [ $? -ne 0 ]; then
+ echo "Cherry-pick failed for $HOTFIX_COMMIT on $HOTFIX_BRANCH. Aborting..."
+ git cherry-pick --abort
+ continue
+ fi
+
+ # Push the hotfix branch to the remote
+ echo "Pushing $HOTFIX_BRANCH..."
+ git push origin "$HOTFIX_BRANCH"
+ echo "Hotfix branch $HOTFIX_BRANCH created and pushed."
+
+ # Check if PR already exists
+ EXISTING_PR=$(gh pr list --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --state open --json number --jq '.[0].number')
+
+ if [ -n "$EXISTING_PR" ]; then
+ echo "An open PR already exists: #$EXISTING_PR. Skipping..."
+ continue
+ fi
+
+ # Create a new PR and capture the output
+ PR_OUTPUT=$(gh pr create --title "Merge $HOTFIX_BRANCH into $RELEASE_BRANCH" \
+ --body "Automated PR to merge \`$HOTFIX_BRANCH\` into \`$RELEASE_BRANCH\`." \
+ --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH")
+
+ # Extract the URL from the output
+ PR_URL=$(echo "$PR_OUTPUT" | grep -Eo 'https://github.com/[^ ]+')
+ echo "Pull request created: $PR_URL"
+
+ # Extract PR number from URL
+ PR_NUMBER=$(basename "$PR_URL")
+ echo "Pull request created: $PR_NUMBER"
+
+ if [ "$AUTO_MERGE" == "true" ]; then
+ echo "Attempting to merge pull request #$PR_NUMBER"
+
+ # Attempt to merge the PR
+ gh pr merge "$PR_NUMBER" --merge --auto --delete-branch
+
+ if [ $? -eq 0 ]; then
+ echo "Pull request #$PR_NUMBER merged successfully."
+ else
+ # Optionally, handle the error or continue
+ echo "Failed to merge pull request #$PR_NUMBER."
+ fi
+ fi
+ done
\ No newline at end of file
diff --git a/.github/workflows/nightly-close-stale-issues.yml b/.github/workflows/nightly-close-stale-issues.yml
new file mode 100644
index 00000000000..a7d296e0a92
--- /dev/null
+++ b/.github/workflows/nightly-close-stale-issues.yml
@@ -0,0 +1,23 @@
+name: 'Nightly - Close stale issues and PRs'
+on:
+ schedule:
+ - cron: '0 11 * * *' # Runs every day at 3 AM PST / 4 AM PDT / 11 AM UTC
+
+permissions:
+ # contents: write # only for delete-branch option
+ issues: write
+ pull-requests: write
+
+jobs:
+ stale:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/stale@v9
+ with:
+ stale-issue-message: 'This issue is stale because it has been open 75 days with no activity. Remove stale label or comment or this will be closed in 15 days.'
+ stale-pr-message: 'This PR is stale because it has been open 75 days with no activity. Remove stale label or comment or this will be closed in 15 days.'
+ close-issue-message: 'This issue was closed because it has been stalled for 90 days with no activity.'
+ close-pr-message: 'This PR was closed because it has been stalled for 90 days with no activity.'
+ days-before-stale: 75
+# days-before-close: 90 # uncomment after we test stale behavior
+
\ No newline at end of file
diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
new file mode 100644
index 00000000000..9aa7030e0b9
--- /dev/null
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -0,0 +1,76 @@
+# Scan for problematic software licenses
+
+# trivy has their own rate limiting issues causing this action to flake
+# we worked around it by hardcoding to different db repos in env
+# can re-enable when they figure it out
+# https://github.com/aquasecurity/trivy/discussions/7538
+# https://github.com/aquasecurity/trivy-action/issues/389
+
+name: 'Nightly - Scan licenses'
+on:
+# schedule:
+# - cron: '0 14 * * *' # Runs every day at 6 AM PST / 7 AM PDT / 2 PM UTC
+ workflow_dispatch: # Allows manual triggering
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+jobs:
+ scan-licenses:
+ # See https://runs-on.com/runners/linux/
+ runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ cache: 'pip'
+ cache-dependency-path: |
+ backend/requirements/default.txt
+ backend/requirements/dev.txt
+ backend/requirements/model_server.txt
+
+ - name: Get explicit and transitive dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
+ pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+ pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
+ pip freeze > requirements-all.txt
+
+ - name: Check python
+ id: license_check_report
+ uses: pilosus/action-pip-license-checker@v2
+ with:
+ requirements: 'requirements-all.txt'
+ fail: 'Copyleft'
+ exclude: '(?i)^(pylint|aio[-_]*).*'
+
+ - name: Print report
+ if: ${{ always() }}
+ run: echo "${{ steps.license_check_report.outputs.report }}"
+
+ - name: Install npm dependencies
+ working-directory: ./web
+ run: npm ci
+
+ - name: Run Trivy vulnerability scanner in repo mode
+ uses: aquasecurity/trivy-action@0.28.0
+ with:
+ scan-type: fs
+ scanners: license
+ format: table
+# format: sarif
+# output: trivy-results.sarif
+ severity: HIGH,CRITICAL
+
+# - name: Upload Trivy scan results to GitHub Security tab
+# uses: github/codeql-action/upload-sarif@v3
+# with:
+# sarif_file: trivy-results.sarif
diff --git a/.github/workflows/run-it.yml b/.github/workflows/pr-Integration-tests.yml
similarity index 50%
rename from .github/workflows/run-it.yml
rename to .github/workflows/pr-Integration-tests.yml
index cbb9954e918..1f28866d6ee 100644
--- a/.github/workflows/run-it.yml
+++ b/.github/workflows/pr-Integration-tests.yml
@@ -1,4 +1,4 @@
-name: Run Integration Tests
+name: Run Integration Tests v2
concurrency:
group: Run-Integration-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
cancel-in-progress: true
@@ -12,11 +12,12 @@ on:
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
jobs:
integration-tests:
# See https://runs-on.com/runners/linux/
- runs-on: [runs-on,runner=8cpu-linux-x64,ram=32,"run-id=${{ github.run_id }}"]
+ runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -30,25 +31,35 @@ jobs:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- # NOTE: we don't need to build the Web Docker image since it's not used
- # during the IT for now. We have a separate action to verify it builds
- # succesfully
+ # tag every docker image with "test" so that we can spin up the correct set
+ # of images during testing
+
+ # We don't need to build the Web Docker image since it's not yet used
+ # in the integration tests. We have a separate action to verify that it builds
+ # successfully.
- name: Pull Web Docker image
run: |
docker pull danswer/danswer-web-server:latest
- docker tag danswer/danswer-web-server:latest danswer/danswer-web-server:it
+ docker tag danswer/danswer-web-server:latest danswer/danswer-web-server:test
+ # we use the runs-on cache for docker builds
+ # in conjunction with runs-on runners, it has better speed and unlimited caching
+ # https://runs-on.com/caching/s3-cache-for-github-actions/
+ # https://runs-on.com/caching/docker/
+ # https://github.com/moby/buildkit#s3-cache-experimental
+
+ # images are built and run locally for testing purposes. Not pushed.
- name: Build Backend Docker image
uses: ./.github/actions/custom-build-and-push
with:
context: ./backend
file: ./backend/Dockerfile
platforms: linux/amd64
- tags: danswer/danswer-backend:it
- cache-from: type=registry,ref=danswer/danswer-backend:it
- cache-to: |
- type=registry,ref=danswer/danswer-backend:it,mode=max
- type=inline
+ tags: danswer/danswer-backend:test
+ push: false
+ load: true
+ cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+ cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
- name: Build Model Server Docker image
uses: ./.github/actions/custom-build-and-push
@@ -56,32 +67,83 @@ jobs:
context: ./backend
file: ./backend/Dockerfile.model_server
platforms: linux/amd64
- tags: danswer/danswer-model-server:it
- cache-from: type=registry,ref=danswer/danswer-model-server:it
- cache-to: |
- type=registry,ref=danswer/danswer-model-server:it,mode=max
- type=inline
-
+ tags: danswer/danswer-model-server:test
+ push: false
+ load: true
+ cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+ cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
- name: Build integration test Docker image
uses: ./.github/actions/custom-build-and-push
with:
context: ./backend
file: ./backend/tests/integration/Dockerfile
platforms: linux/amd64
- tags: danswer/integration-test-runner:it
- cache-from: type=registry,ref=danswer/integration-test-runner:it
- cache-to: |
- type=registry,ref=danswer/integration-test-runner:it,mode=max
- type=inline
+ tags: danswer/danswer-integration:test
+ push: false
+ load: true
+ cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+ cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+ # Start containers for multi-tenant tests
+ - name: Start Docker containers for multi-tenant tests
+ run: |
+ cd deployment/docker_compose
+ ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
+ MULTI_TENANT=true \
+ AUTH_TYPE=basic \
+ REQUIRE_EMAIL_VERIFICATION=false \
+ DISABLE_TELEMETRY=true \
+ IMAGE_TAG=test \
+ docker compose -f docker-compose.dev.yml -p danswer-stack up -d
+ id: start_docker_multi_tenant
- - name: Start Docker containers
+ # In practice, `cloud` Auth type would require OAUTH credentials to be set.
+ - name: Run Multi-Tenant Integration Tests
+ run: |
+ echo "Running integration tests..."
+ docker run --rm --network danswer-stack_default \
+ --name test-runner \
+ -e POSTGRES_HOST=relational_db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=password \
+ -e POSTGRES_DB=postgres \
+ -e VESPA_HOST=index \
+ -e REDIS_HOST=cache \
+ -e API_SERVER_HOST=api_server \
+ -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+ -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+ -e TEST_WEB_HOSTNAME=test-runner \
+ -e AUTH_TYPE=cloud \
+ -e MULTI_TENANT=true \
+ danswer/danswer-integration:test \
+ /app/tests/integration/multitenant_tests
+ continue-on-error: true
+ id: run_multitenant_tests
+
+ - name: Check multi-tenant test results
+ run: |
+ if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
+ echo "Integration tests failed. Exiting with error."
+ exit 1
+ else
+ echo "All integration tests passed successfully."
+ fi
+
+ - name: Stop multi-tenant Docker containers
+ run: |
+ cd deployment/docker_compose
+ docker compose -f docker-compose.dev.yml -p danswer-stack down -v
+
+
+ - name: Start Docker containers
run: |
cd deployment/docker_compose
ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
AUTH_TYPE=basic \
REQUIRE_EMAIL_VERIFICATION=false \
DISABLE_TELEMETRY=true \
- IMAGE_TAG=it \
+ IMAGE_TAG=test \
docker compose -f docker-compose.dev.yml -p danswer-stack up -d
id: start_docker
@@ -119,7 +181,7 @@ jobs:
done
echo "Finished waiting for service."
- - name: Run integration tests
+ - name: Run Standard Integration Tests
run: |
echo "Running integration tests..."
docker run --rm --network danswer-stack_default \
@@ -132,8 +194,10 @@ jobs:
-e REDIS_HOST=cache \
-e API_SERVER_HOST=api_server \
-e OPENAI_API_KEY=${OPENAI_API_KEY} \
+ -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-e TEST_WEB_HOSTNAME=test-runner \
- danswer/integration-test-runner:it
+ danswer/danswer-integration:test \
+ /app/tests/integration/tests
continue-on-error: true
id: run_tests
@@ -146,6 +210,11 @@ jobs:
echo "All integration tests passed successfully."
fi
+ - name: Stop Docker containers
+ run: |
+ cd deployment/docker_compose
+ docker compose -f docker-compose.dev.yml -p danswer-stack down -v
+
- name: Save Docker logs
if: success() || failure()
run: |
@@ -155,7 +224,7 @@ jobs:
- name: Upload logs
if: success() || failure()
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: docker-logs
path: ${{ github.workspace }}/docker-compose.log
diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
new file mode 100644
index 00000000000..273f00a5c5a
--- /dev/null
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -0,0 +1,124 @@
+name: Backport on Merge
+
+# Note this workflow does not trigger the builds, be sure to manually tag the branches to trigger the builds
+
+on:
+ pull_request:
+ types: [closed] # Later we check for merge so only PRs that go in can get backported
+
+permissions:
+ contents: write
+ actions: write
+
+jobs:
+ backport:
+ if: github.event.pull_request.merged == true
+ runs-on: ubuntu-latest
+ env:
+ GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
+ fetch-depth: 0
+
+ - name: Set up Git user
+ run: |
+ git config user.name "Richard Kuo [bot]"
+ git config user.email "rkuo[bot]@danswer.ai"
+ git fetch --prune
+
+ - name: Check for Backport Checkbox
+ id: checkbox-check
+ run: |
+ PR_BODY="${{ github.event.pull_request.body }}"
+ if [[ "$PR_BODY" == *"[x] This PR should be backported"* ]]; then
+ echo "backport=true" >> $GITHUB_OUTPUT
+ else
+ echo "backport=false" >> $GITHUB_OUTPUT
+ fi
+
+ - name: List and sort release branches
+ id: list-branches
+ run: |
+ git fetch --all --tags
+ BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/release/* | sed 's|origin/release/||' | sort -Vr)
+ BETA=$(echo "$BRANCHES" | head -n 1)
+ STABLE=$(echo "$BRANCHES" | head -n 2 | tail -n 1)
+ echo "beta=release/$BETA" >> $GITHUB_OUTPUT
+ echo "stable=release/$STABLE" >> $GITHUB_OUTPUT
+ # Fetch latest tags for beta and stable
+ LATEST_BETA_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*-beta.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$" | grep -v -- "-cloud" | sort -Vr | head -n 1)
+ LATEST_STABLE_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -Vr | head -n 1)
+
+ # Handle case where no beta tags exist
+ if [[ -z "$LATEST_BETA_TAG" ]]; then
+ NEW_BETA_TAG="v1.0.0-beta.1"
+ else
+ NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 "." $3 "-beta." ($NF+1)}')
+ fi
+
+ # Increment latest stable tag
+ NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}')
+ echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT
+ echo "latest_stable_tag=$LATEST_STABLE_TAG" >> $GITHUB_OUTPUT
+ echo "new_beta_tag=$NEW_BETA_TAG" >> $GITHUB_OUTPUT
+ echo "new_stable_tag=$NEW_STABLE_TAG" >> $GITHUB_OUTPUT
+
+ - name: Echo branch and tag information
+ run: |
+ echo "Beta branch: ${{ steps.list-branches.outputs.beta }}"
+ echo "Stable branch: ${{ steps.list-branches.outputs.stable }}"
+ echo "Latest beta tag: ${{ steps.list-branches.outputs.latest_beta_tag }}"
+ echo "Latest stable tag: ${{ steps.list-branches.outputs.latest_stable_tag }}"
+ echo "New beta tag: ${{ steps.list-branches.outputs.new_beta_tag }}"
+ echo "New stable tag: ${{ steps.list-branches.outputs.new_stable_tag }}"
+
+ - name: Trigger Backport
+ if: steps.checkbox-check.outputs.backport == 'true'
+ run: |
+ set -e
+ echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
+
+ # Echo the merge commit SHA
+ echo "Merge commit SHA: ${{ github.event.pull_request.merge_commit_sha }}"
+
+ # Fetch all history for all branches and tags
+ git fetch --prune
+
+ # Reset and prepare the beta branch
+ git checkout ${{ steps.list-branches.outputs.beta }}
+ echo "Last 5 commits on beta branch:"
+ git log -n 5 --pretty=format:"%H"
+ echo "" # Newline for formatting
+
+ # Cherry-pick the merge commit from the merged PR
+ git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
+ echo "Cherry-pick to beta failed due to conflicts."
+ exit 1
+ }
+
+ # Create new beta branch/tag
+ git tag ${{ steps.list-branches.outputs.new_beta_tag }}
+ # Push the changes and tag to the beta branch using PAT
+ git push origin ${{ steps.list-branches.outputs.beta }}
+ git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
+
+ # Reset and prepare the stable branch
+ git checkout ${{ steps.list-branches.outputs.stable }}
+ echo "Last 5 commits on stable branch:"
+ git log -n 5 --pretty=format:"%H"
+ echo "" # Newline for formatting
+
+ # Cherry-pick the merge commit from the merged PR
+ git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
+ echo "Cherry-pick to stable failed due to conflicts."
+ exit 1
+ }
+
+ # Create new stable branch/tag
+ git tag ${{ steps.list-branches.outputs.new_stable_tag }}
+ # Push the changes and tag to the stable branch using PAT
+ git push origin ${{ steps.list-branches.outputs.stable }}
+ git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
diff --git a/.github/workflows/pr-python-checks.yml b/.github/workflows/pr-python-checks.yml
index 0a9e9f96a63..db16848bd2f 100644
--- a/.github/workflows/pr-python-checks.yml
+++ b/.github/workflows/pr-python-checks.yml
@@ -14,10 +14,10 @@ jobs:
steps:
- name: Checkout code
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
diff --git a/.github/workflows/pr-python-connector-tests.yml b/.github/workflows/pr-python-connector-tests.yml
index 642618000d2..88053427726 100644
--- a/.github/workflows/pr-python-connector-tests.yml
+++ b/.github/workflows/pr-python-connector-tests.yml
@@ -18,6 +18,11 @@ env:
# Jira
JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+ # Google
+ GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR }}
+ GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }}
+ GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR }}
+ GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR }}
jobs:
connectors-check:
@@ -32,7 +37,7 @@ jobs:
uses: actions/checkout@v4
- name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: "3.11"
cache: "pip"
diff --git a/.github/workflows/pr-python-model-tests.yml b/.github/workflows/pr-python-model-tests.yml
new file mode 100644
index 00000000000..a070eea27a7
--- /dev/null
+++ b/.github/workflows/pr-python-model-tests.yml
@@ -0,0 +1,58 @@
+name: Connector Tests
+
+on:
+ schedule:
+ # This cron expression runs the job daily at 16:00 UTC (9am PT)
+ - cron: "0 16 * * *"
+
+env:
+ # Bedrock
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
+
+ # OpenAI
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+jobs:
+ model-check:
+ # See https://runs-on.com/runners/linux/
+ runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]
+
+ env:
+ PYTHONPATH: ./backend
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+ cache: "pip"
+ cache-dependency-path: |
+ backend/requirements/default.txt
+ backend/requirements/dev.txt
+
+ - name: Install Dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
+ pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+
+ - name: Run Tests
+ shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
+ run: |
+ py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/llm
+ py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/embedding
+
+ - name: Alert on Failure
+ if: failure() && github.event_name == 'schedule'
+ env:
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+ run: |
+ curl -X POST \
+ -H 'Content-type: application/json' \
+ --data '{"text":"Scheduled Model Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
+ $SLACK_WEBHOOK
diff --git a/.github/workflows/pr-python-tests.yml b/.github/workflows/pr-python-tests.yml
index ce57a7a5814..5637300615b 100644
--- a/.github/workflows/pr-python-tests.yml
+++ b/.github/workflows/pr-python-tests.yml
@@ -21,7 +21,7 @@ jobs:
uses: actions/checkout@v4
- name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
diff --git a/.github/workflows/pr-quality-checks.yml b/.github/workflows/pr-quality-checks.yml
index 128317a79ce..3ba206669a6 100644
--- a/.github/workflows/pr-quality-checks.yml
+++ b/.github/workflows/pr-quality-checks.yml
@@ -18,6 +18,6 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- - uses: pre-commit/action@v3.0.0
+ - uses: pre-commit/action@v3.0.1
with:
extra_args: ${{ github.event_name == 'pull_request' && format('--from-ref {0} --to-ref {1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) || '' }}
diff --git a/.github/workflows/tag-nightly.yml b/.github/workflows/tag-nightly.yml
index bf2699d9fd4..50bb20808a3 100644
--- a/.github/workflows/tag-nightly.yml
+++ b/.github/workflows/tag-nightly.yml
@@ -2,7 +2,7 @@ name: Nightly Tag Push
on:
schedule:
- - cron: '0 0 * * *' # Runs every day at midnight UTC
+ - cron: '0 10 * * *' # Runs every day at 2 AM PST / 3 AM PDT / 10 AM UTC
permissions:
contents: write # Allows pushing tags to the repository
diff --git a/.vscode/launch.template.jsonc b/.vscode/launch.template.jsonc
index c733800981c..87875907cd5 100644
--- a/.vscode/launch.template.jsonc
+++ b/.vscode/launch.template.jsonc
@@ -6,19 +6,69 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"compounds": [
+ {
+ // Dummy entry used to label the group
+ "name": "--- Compound ---",
+ "configurations": [
+ "--- Individual ---"
+ ],
+ "presentation": {
+ "group": "1",
+ }
+ },
{
"name": "Run All Danswer Services",
"configurations": [
"Web Server",
"Model Server",
"API Server",
- "Indexing",
- "Background Jobs",
- "Slack Bot"
- ]
- }
+ "Slack Bot",
+ "Celery primary",
+ "Celery light",
+ "Celery heavy",
+ "Celery indexing",
+ "Celery beat",
+ ],
+ "presentation": {
+ "group": "1",
+ }
+ },
+ {
+ "name": "Web / Model / API",
+ "configurations": [
+ "Web Server",
+ "Model Server",
+ "API Server",
+ ],
+ "presentation": {
+ "group": "1",
+ }
+ },
+ {
+ "name": "Celery (all)",
+ "configurations": [
+ "Celery primary",
+ "Celery light",
+ "Celery heavy",
+ "Celery indexing",
+ "Celery beat"
+ ],
+ "presentation": {
+ "group": "1",
+ }
+ }
],
"configurations": [
+ {
+ // Dummy entry used to label the group
+ "name": "--- Individual ---",
+ "type": "node",
+ "request": "launch",
+ "presentation": {
+ "group": "2",
+ "order": 0
+ }
+ },
{
"name": "Web Server",
"type": "node",
@@ -29,7 +79,11 @@
"runtimeArgs": [
"run", "dev"
],
- "console": "integratedTerminal"
+ "presentation": {
+ "group": "2",
+ },
+ "console": "integratedTerminal",
+ "consoleTitle": "Web Server Console"
},
{
"name": "Model Server",
@@ -48,7 +102,11 @@
"--reload",
"--port",
"9000"
- ]
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Model Server Console"
},
{
"name": "API Server",
@@ -68,57 +126,171 @@
"--reload",
"--port",
"8080"
- ]
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "API Server Console"
},
+ // For the listener to access the Slack API,
+ // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
{
- "name": "Indexing",
- "consoleName": "Indexing",
+ "name": "Slack Bot",
+ "consoleName": "Slack Bot",
"type": "debugpy",
"request": "launch",
- "program": "danswer/background/update.py",
+ "program": "danswer/danswerbot/slack/listener.py",
"cwd": "${workspaceFolder}/backend",
"envFile": "${workspaceFolder}/.vscode/.env",
"env": {
- "ENABLE_MULTIPASS_INDEXING": "false",
"LOG_LEVEL": "DEBUG",
"PYTHONUNBUFFERED": "1",
"PYTHONPATH": "."
- }
+ },
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Slack Bot Console"
},
- // Celery and all async jobs, usually would include indexing as well but this is handled separately above for dev
{
- "name": "Background Jobs",
- "consoleName": "Background Jobs",
+ "name": "Celery primary",
"type": "debugpy",
"request": "launch",
- "program": "scripts/dev_run_background_jobs.py",
+ "module": "celery",
"cwd": "${workspaceFolder}/backend",
"envFile": "${workspaceFolder}/.vscode/.env",
"env": {
- "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+ "LOG_LEVEL": "INFO",
+ "PYTHONUNBUFFERED": "1",
+ "PYTHONPATH": "."
+ },
+ "args": [
+ "-A",
+ "danswer.background.celery.versioned_apps.primary",
+ "worker",
+ "--pool=threads",
+ "--concurrency=4",
+ "--prefetch-multiplier=1",
+ "--loglevel=INFO",
+ "--hostname=primary@%n",
+ "-Q",
+ "celery",
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Celery primary Console"
+ },
+ {
+ "name": "Celery light",
+ "type": "debugpy",
+ "request": "launch",
+ "module": "celery",
+ "cwd": "${workspaceFolder}/backend",
+ "envFile": "${workspaceFolder}/.vscode/.env",
+ "env": {
+ "LOG_LEVEL": "INFO",
+ "PYTHONUNBUFFERED": "1",
+ "PYTHONPATH": "."
+ },
+ "args": [
+ "-A",
+ "danswer.background.celery.versioned_apps.light",
+ "worker",
+ "--pool=threads",
+ "--concurrency=64",
+ "--prefetch-multiplier=8",
+ "--loglevel=INFO",
+ "--hostname=light@%n",
+ "-Q",
+ "vespa_metadata_sync,connector_deletion",
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Celery light Console"
+ },
+ {
+ "name": "Celery heavy",
+ "type": "debugpy",
+ "request": "launch",
+ "module": "celery",
+ "cwd": "${workspaceFolder}/backend",
+ "envFile": "${workspaceFolder}/.vscode/.env",
+ "env": {
+ "LOG_LEVEL": "INFO",
+ "PYTHONUNBUFFERED": "1",
+ "PYTHONPATH": "."
+ },
+ "args": [
+ "-A",
+ "danswer.background.celery.versioned_apps.heavy",
+ "worker",
+ "--pool=threads",
+ "--concurrency=4",
+ "--prefetch-multiplier=1",
+ "--loglevel=INFO",
+ "--hostname=heavy@%n",
+ "-Q",
+ "connector_pruning",
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Celery heavy Console"
+ },
+ {
+ "name": "Celery indexing",
+ "type": "debugpy",
+ "request": "launch",
+ "module": "celery",
+ "cwd": "${workspaceFolder}/backend",
+ "envFile": "${workspaceFolder}/.vscode/.env",
+ "env": {
+ "ENABLE_MULTIPASS_INDEXING": "false",
"LOG_LEVEL": "DEBUG",
"PYTHONUNBUFFERED": "1",
"PYTHONPATH": "."
},
"args": [
- "--no-indexing"
- ]
+ "-A",
+ "danswer.background.celery.versioned_apps.indexing",
+ "worker",
+ "--pool=threads",
+ "--concurrency=1",
+ "--prefetch-multiplier=1",
+ "--loglevel=INFO",
+ "--hostname=indexing@%n",
+ "-Q",
+ "connector_indexing",
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Celery indexing Console"
},
- // For the listner to access the Slack API,
- // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
{
- "name": "Slack Bot",
- "consoleName": "Slack Bot",
+ "name": "Celery beat",
"type": "debugpy",
"request": "launch",
- "program": "danswer/danswerbot/slack/listener.py",
+ "module": "celery",
"cwd": "${workspaceFolder}/backend",
"envFile": "${workspaceFolder}/.vscode/.env",
"env": {
"LOG_LEVEL": "DEBUG",
"PYTHONUNBUFFERED": "1",
"PYTHONPATH": "."
- }
+ },
+ "args": [
+ "-A",
+ "danswer.background.celery.versioned_apps.beat",
+ "beat",
+ "--loglevel=INFO",
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Celery beat Console"
},
{
"name": "Pytest",
@@ -137,8 +309,22 @@
"-v"
// Specify a sepcific module/test to run or provide nothing to run all tests
//"tests/unit/danswer/llm/answering/test_prune_and_merge.py"
- ]
+ ],
+ "presentation": {
+ "group": "2",
+ },
+ "consoleTitle": "Pytest Console"
},
+ {
+ // Dummy entry used to label the group
+ "name": "--- Tasks ---",
+ "type": "node",
+ "request": "launch",
+ "presentation": {
+ "group": "3",
+ "order": 0
+ }
+ },
{
"name": "Clear and Restart External Volumes and Containers",
"type": "node",
@@ -147,7 +333,27 @@
"runtimeArgs": ["${workspaceFolder}/backend/scripts/restart_containers.sh"],
"cwd": "${workspaceFolder}",
"console": "integratedTerminal",
- "stopOnEntry": true
- }
+ "stopOnEntry": true,
+ "presentation": {
+ "group": "3",
+ },
+ },
+ {
+ // Celery jobs launched through a single background script (legacy)
+ // Recommend using the "Celery (all)" compound launch instead.
+ "name": "Background Jobs",
+ "consoleName": "Background Jobs",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "scripts/dev_run_background_jobs.py",
+ "cwd": "${workspaceFolder}/backend",
+ "envFile": "${workspaceFolder}/.vscode/.env",
+ "env": {
+ "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+ "LOG_LEVEL": "DEBUG",
+ "PYTHONUNBUFFERED": "1",
+ "PYTHONPATH": "."
+ },
+ },
]
}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3e4415188a1..779cabbb491 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -22,7 +22,7 @@ Your input is vital to making sure that Danswer moves in the right direction.
Before starting on implementation, please raise a GitHub issue.
And always feel free to message us (Chris Weaver / Yuhong Sun) on
-[Slack](https://join.slack.com/t/danswer/shared_invite/zt-2afut44lv-Rw3kSWu6_OmdAXRpCv80DQ) /
+[Slack](https://join.slack.com/t/danswer/shared_invite/zt-2lcmqw703-071hBuZBfNEOGUsLa5PXvQ) /
[Discord](https://discord.gg/TDJ59cGV2X) directly about anything at all.
diff --git a/README.md b/README.md
index aff3cd57d5a..0cc95ff4464 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
+
@@ -68,13 +69,13 @@ We also have built-in support for deployment on Kubernetes. Files for that can b
## 🚧 Roadmap
* Chat/Prompt sharing with specific teammates and user groups.
-* Multi-Model model support, chat with images, video etc.
+* Multimodal model support, chat with images, video etc.
* Choosing between LLMs and parameters during chat session.
* Tool calling and agent configurations options.
* Organizational understanding and ability to locate and suggest experts from your team.
-## Other Noteable Benefits of Danswer
+## Other Notable Benefits of Danswer
* User Authentication with document level access management.
* Best in class Hybrid Search across all sources (BM-25 + prefix aware embedding models).
* Admin Dashboard to configure connectors, document-sets, access, etc.
@@ -127,3 +128,19 @@ To try the Danswer Enterprise Edition:
## 💡 Contributing
Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
+
+## ⭐Star History
+
+[](https://star-history.com/#danswer-ai/danswer&Date)
+
+## ✨Contributors
+
+
+
+
+
+
- This icon will visually represent your Assistant
-
+ This icon will visually represent your Assistant
@@ -648,13 +599,7 @@ export function AssistantEditor({
placeholder="e.g. 'You are a professional email writing assistant that always uses a polite enthusiastic tone, emphasizes action items, and leaves blanks for the human to fill in when you have unknowns'"
onChange={(e) => {
setFieldValue("system_prompt", e.target.value);
- triggerFinalPromptUpdate(
- e.target.value,
- values.task_prompt,
- searchToolEnabled()
- );
}}
- error={finalPromptError}
/>
@@ -662,16 +607,14 @@ export function AssistantEditor({
Default AI Model{" "}
-
+
-
- Select a Large Language Model (Generative AI model) to
- power this Assistant
-
+ Select a Large Language Model (Generative AI model) to
+ power this Assistant
@@ -681,7 +624,10 @@ export function AssistantEditor({
otherwise specified below.
{admin &&
user?.preferences.default_model &&
- ` Your current (user-specific) default model is ${getDisplayNameForModel(destructureValue(user?.preferences?.default_model!).modelName)}`}
+ ` Your current (user-specific) default model is ${getDisplayNameForModel(
+ destructureValue(user?.preferences?.default_model!)
+ .modelName
+ )}`}
{admin ? (
@@ -756,16 +702,14 @@ export function AssistantEditor({
Capabilities{" "}
-
+
-
- You can give your assistant advanced capabilities like
- image generation
-
+ You can give your assistant advanced capabilities like
+ image generation
@@ -776,12 +720,13 @@ export function AssistantEditor({
+
+ Add pre-defined messages to help users get started. Only
+ the first 4 will be displayed.
+
Add New
@@ -1208,9 +1162,7 @@ export function AssistantEditor({
)
@@ -68,13 +69,13 @@ We also have built-in support for deployment on Kubernetes. Files for that can b
## 🚧 Roadmap
* Chat/Prompt sharing with specific teammates and user groups.
-* Multi-Model model support, chat with images, video etc.
+* Multimodal model support, chat with images, video etc.
* Choosing between LLMs and parameters during chat session.
* Tool calling and agent configurations options.
* Organizational understanding and ability to locate and suggest experts from your team.
-## Other Noteable Benefits of Danswer
+## Other Notable Benefits of Danswer
* User Authentication with document level access management.
* Best in class Hybrid Search across all sources (BM-25 + prefix aware embedding models).
* Admin Dashboard to configure connectors, document-sets, access, etc.
@@ -127,3 +128,19 @@ To try the Danswer Enterprise Edition:
## 💡 Contributing
Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
+
+## ⭐Star History
+
+[](https://star-history.com/#danswer-ai/danswer&Date)
+
+## ✨Contributors
+
+
+