diff --git a/.github/workflows/backwards_compatibility_marqo_orchestrator.yml b/.github/workflows/backwards_compatibility_marqo_orchestrator.yml index 07b563aec..3848c5f5a 100644 --- a/.github/workflows/backwards_compatibility_marqo_orchestrator.yml +++ b/.github/workflows/backwards_compatibility_marqo_orchestrator.yml @@ -137,7 +137,7 @@ jobs: run-backwards-compatibility-execution-workflow: # Job to trigger execution workflows for backwards compatibility test for each version combination - name: Run all backwards compatibility tests execution workflows + name: ${{ matrix.from_version }} Run all backwards compatibility tests execution workflows needs: [orchestrate, check-if-image-exists, build-and-push-image] if: always() && (needs.orchestrate.result == 'success') strategy: @@ -164,7 +164,7 @@ jobs: run-rollback-tests-execution-workflow: # Job to trigger execution workflows for rollback test for each version combination - name: Run all rollback tests execution workflows + name: ${{ matrix.from_version }} Run all rollback tests execution workflows needs: [ orchestrate, check-if-image-exists, build-and-push-image ] if: always() && (needs.orchestrate.result == 'success') strategy: diff --git a/.github/workflows/unit_test_200gb_CI.yml b/.github/workflows/unit_test_200gb_CI.yml index a8a3b4f4e..1ad6c8e7d 100644 --- a/.github/workflows/unit_test_200gb_CI.yml +++ b/.github/workflows/unit_test_200gb_CI.yml @@ -8,6 +8,8 @@ on: branches: - mainline - releases/* + paths-ignore: + - '**.md' pull_request: branches: - mainline @@ -21,9 +23,45 @@ permissions: contents: read jobs: + Check-Changes: + runs-on: ubuntu-latest + outputs: + doc_only: ${{ steps.set-output.outputs.doc_only }} + steps: + - name: Checkout marqo repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: marqo + + - name: Check for Documentation-Only Changes + id: set-output + run: | + cd marqo + set -x + # Get the base and head commits from the GitHub event + BASE_COMMIT=${{ github.event.pull_request.base.sha }} + HEAD_COMMIT=${{ github.event.pull_request.head.sha }} + + # Debug: Print base and head commits + echo "Base commit: $BASE_COMMIT" + echo "Head commit: $HEAD_COMMIT" + + # Perform the diff to check for non-documentation changes + if git diff --name-only $BASE_COMMIT $HEAD_COMMIT -- | grep -vE '\.(md)$'; then + echo "doc_only=false" >> $GITHUB_ENV + echo "doc_only=false" >> $GITHUB_OUTPUT + else + echo "doc_only=true" >> $GITHUB_ENV + echo "doc_only=true" >> $GITHUB_OUTPUT + fi + Start-Runner: name: Start self-hosted EC2 runner runs-on: ubuntu-latest + needs: + - Check-Changes + if: ${{ needs.Check-Changes.outputs.doc_only == 'false' }} # Run only if there are non-documentation changes outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -56,7 +94,10 @@ jobs: Test-Marqo: name: Run Unit Tests - needs: Start-Runner # required to start the main job when the runner is ready + needs: + - Check-Changes # required to start the main job when the runner is ready + - Start-Runner # required to get output from the start-runner job + if: ${{ needs.Check-Changes.outputs.doc_only == 'false' }} # Run only if there are non-documentation changes runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner environment: marqo-test-suite steps: @@ -170,7 +211,7 @@ jobs: export PYTHONPATH="./tests:./src:." set -o pipefail pytest --ignore=tests/test_documentation.py --ignore=tests/compatibility_tests \ - --durations=100 --cov=src --cov-branch --cov-context=test \ + --durations=100 --cov=src --cov-branch --cov-context=test --cov-fail-under=69 \ --cov-report=html:cov_html --cov-report=xml:cov.xml --cov-report term:skip-covered \ --md-report --md-report-flavor gfm --md-report-output pytest_result_summary.md \ tests | tee pytest_output.txt @@ -185,7 +226,7 @@ jobs: echo "Running diff-cover against branch $BASE_BRANCH" git fetch origin $BASE_BRANCH:$BASE_BRANCH diff-cover cov.xml --html-report diff_cov.html --markdown-report diff_cov.md \ - --compare-branch $BASE_BRANCH + --compare-branch $BASE_BRANCH --fail-under=95 else echo "Skipping diff-cover on Push events" echo "Skipped diff-cover on Push events" > marqo/diff_cov.md @@ -226,8 +267,9 @@ jobs: needs: - Start-Runner # required to get output from the start-runner job - Test-Marqo # required to wait when the main job is done + - Check-Changes runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + if: ${{ needs.start-runner.outputs.label }} # Only stop the runner if it was started steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/RELEASE.md b/RELEASE.md index 58f660948..246a2e465 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,21 @@ +# Release 2.15.0 + +## New Features +- Global Score Modifiers for Hybrid Search ([#1082](https://github.com/marqo-ai/marqo/pull/1082)). Introduce global score modifiers for hybrid search, allowing fine-tuned adjustments to RRF scores in combined result lists. This enhancement provides better control over returned results in hybrid search scenarios. Use the `rerankDepth` parameter to control the number of hits to rerank. For detailed usage, check [here](https://docs.marqo.ai/latest/reference/api/search/search/#rerank-depth). + +- Custom LanguageBind Model ([#1072](https://github.com/marqo-ai/marqo/pull/1072)). Marqo now supports loading custom LanguageBind models from S3 buckets, URLs, or HuggingFace model cards. Fine-tune your own LanguageBind model and integrate it with Marqo to achieve better in-domain results. For more details, see [here](https://docs.marqo.ai/2.15/models/marqo/bring-your-own-model/#bring-your-own-languagebind-models). + +- Add Marqtune models to the model registry ([#1063](https://github.com/marqo-ai/marqo/pull/1063)). Marqtune models have been added to the model registry, with some models renamed to align with the Marqtune naming convention. This update improves consistency and makes it easier to identify models Marqo and Marqtune. The changes are fully backwards compatible. + +## Bug Fixes and Minor Changes +- Fix a bug where Marqo incorrectly inferred the modality of a field, even when the field is not a tensor field in an unstructured index ([#1086](https://github.com/marqo-ai/marqo/pull/1086)). +- Resolve an issue where Languagebind models could only handle a single video or audio in a weighted search query. You can now provide multiple videos and audios in a weighted query seamlessly.([#1072](https://github.com/marqo-ai/marqo/pull/1072)). +- Improve memory usage when indexing image documents with LanguageBind models, enabling more efficient handling of image data. ([#1072](https://github.com/marqo-ai/marqo/pull/1072)). + +## Contributor Shout-Outs +- A huge thanks to our 4.7k stargazers for your continued support! +- Thanks a lot for the discussion and suggestions in our community. Join us on [Slack](https://join.slack.com/t/marqo-community/shared_invite/zt-2jm456s90-1pFxdE5kDQt5imqddXUIcw) and our [forum](https://community.marqo.ai/) today! + # Release 2.14.1 ## New features