From 704a34c714ac57a0bb0cddd6301aa84dc7f6c4f2 Mon Sep 17 00:00:00 2001 From: Aditya Bharadwaj Date: Mon, 20 Jan 2025 11:53:52 +1100 Subject: [PATCH 1/4] Changing backwards compatibility GitHub actions UI to show version at the start (#1092) --- .../workflows/backwards_compatibility_marqo_orchestrator.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backwards_compatibility_marqo_orchestrator.yml b/.github/workflows/backwards_compatibility_marqo_orchestrator.yml index 07b563aec..3848c5f5a 100644 --- a/.github/workflows/backwards_compatibility_marqo_orchestrator.yml +++ b/.github/workflows/backwards_compatibility_marqo_orchestrator.yml @@ -137,7 +137,7 @@ jobs: run-backwards-compatibility-execution-workflow: # Job to trigger execution workflows for backwards compatibility test for each version combination - name: Run all backwards compatibility tests execution workflows + name: ${{ matrix.from_version }} Run all backwards compatibility tests execution workflows needs: [orchestrate, check-if-image-exists, build-and-push-image] if: always() && (needs.orchestrate.result == 'success') strategy: @@ -164,7 +164,7 @@ jobs: run-rollback-tests-execution-workflow: # Job to trigger execution workflows for rollback test for each version combination - name: Run all rollback tests execution workflows + name: ${{ matrix.from_version }} Run all rollback tests execution workflows needs: [ orchestrate, check-if-image-exists, build-and-push-image ] if: always() && (needs.orchestrate.result == 'success') strategy: From 6edba557c8fb49decfe61bab75c7f9f3d9bb49f8 Mon Sep 17 00:00:00 2001 From: Yihan Zhao Date: Wed, 22 Jan 2025 11:18:26 +1100 Subject: [PATCH 2/4] Fail build pipeline if code coverage is under threshold (#1093) --- .github/workflows/unit_test_200gb_CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_test_200gb_CI.yml b/.github/workflows/unit_test_200gb_CI.yml index a8a3b4f4e..fd5924a4b 100644 --- a/.github/workflows/unit_test_200gb_CI.yml +++ b/.github/workflows/unit_test_200gb_CI.yml @@ -170,7 +170,7 @@ jobs: export PYTHONPATH="./tests:./src:." set -o pipefail pytest --ignore=tests/test_documentation.py --ignore=tests/compatibility_tests \ - --durations=100 --cov=src --cov-branch --cov-context=test \ + --durations=100 --cov=src --cov-branch --cov-context=test --cov-fail-under=69 \ --cov-report=html:cov_html --cov-report=xml:cov.xml --cov-report term:skip-covered \ --md-report --md-report-flavor gfm --md-report-output pytest_result_summary.md \ tests | tee pytest_output.txt @@ -185,7 +185,7 @@ jobs: echo "Running diff-cover against branch $BASE_BRANCH" git fetch origin $BASE_BRANCH:$BASE_BRANCH diff-cover cov.xml --html-report diff_cov.html --markdown-report diff_cov.md \ - --compare-branch $BASE_BRANCH + --compare-branch $BASE_BRANCH --fail-under=95 else echo "Skipping diff-cover on Push events" echo "Skipped diff-cover on Push events" > marqo/diff_cov.md From 2766332d73b14860be08b24ddd04b373121dc595 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 22 Jan 2025 14:45:29 +1100 Subject: [PATCH 3/4] Add 2.15 release notes (#1094) --- RELEASE.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 58f660948..246a2e465 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,21 @@ +# Release 2.15.0 + +## New Features +- Global Score Modifiers for Hybrid Search ([#1082](https://github.com/marqo-ai/marqo/pull/1082)). Introduce global score modifiers for hybrid search, allowing fine-tuned adjustments to RRF scores in combined result lists. This enhancement provides better control over returned results in hybrid search scenarios. Use the `rerankDepth` parameter to control the number of hits to rerank. For detailed usage, check [here](https://docs.marqo.ai/latest/reference/api/search/search/#rerank-depth). + +- Custom LanguageBind Model ([#1072](https://github.com/marqo-ai/marqo/pull/1072)). Marqo now supports loading custom LanguageBind models from S3 buckets, URLs, or HuggingFace model cards. Fine-tune your own LanguageBind model and integrate it with Marqo to achieve better in-domain results. For more details, see [here](https://docs.marqo.ai/2.15/models/marqo/bring-your-own-model/#bring-your-own-languagebind-models). + +- Add Marqtune models to the model registry ([#1063](https://github.com/marqo-ai/marqo/pull/1063)). Marqtune models have been added to the model registry, with some models renamed to align with the Marqtune naming convention. This update improves consistency and makes it easier to identify models Marqo and Marqtune. The changes are fully backwards compatible. + +## Bug Fixes and Minor Changes +- Fix a bug where Marqo incorrectly inferred the modality of a field, even when the field is not a tensor field in an unstructured index ([#1086](https://github.com/marqo-ai/marqo/pull/1086)). +- Resolve an issue where Languagebind models could only handle a single video or audio in a weighted search query. You can now provide multiple videos and audios in a weighted query seamlessly.([#1072](https://github.com/marqo-ai/marqo/pull/1072)). +- Improve memory usage when indexing image documents with LanguageBind models, enabling more efficient handling of image data. ([#1072](https://github.com/marqo-ai/marqo/pull/1072)). + +## Contributor Shout-Outs +- A huge thanks to our 4.7k stargazers for your continued support! +- Thanks a lot for the discussion and suggestions in our community. Join us on [Slack](https://join.slack.com/t/marqo-community/shared_invite/zt-2jm456s90-1pFxdE5kDQt5imqddXUIcw) and our [forum](https://community.marqo.ai/) today! + # Release 2.14.1 ## New features From 693b1c7561e3419e623314456e37a1b3a43e1c92 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Thu, 23 Jan 2025 17:35:19 +1100 Subject: [PATCH 4/4] Skip unit tests when only making documentation changes (#1095) --- .github/workflows/unit_test_200gb_CI.yml | 46 ++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_test_200gb_CI.yml b/.github/workflows/unit_test_200gb_CI.yml index fd5924a4b..1ad6c8e7d 100644 --- a/.github/workflows/unit_test_200gb_CI.yml +++ b/.github/workflows/unit_test_200gb_CI.yml @@ -8,6 +8,8 @@ on: branches: - mainline - releases/* + paths-ignore: + - '**.md' pull_request: branches: - mainline @@ -21,9 +23,45 @@ permissions: contents: read jobs: + Check-Changes: + runs-on: ubuntu-latest + outputs: + doc_only: ${{ steps.set-output.outputs.doc_only }} + steps: + - name: Checkout marqo repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: marqo + + - name: Check for Documentation-Only Changes + id: set-output + run: | + cd marqo + set -x + # Get the base and head commits from the GitHub event + BASE_COMMIT=${{ github.event.pull_request.base.sha }} + HEAD_COMMIT=${{ github.event.pull_request.head.sha }} + + # Debug: Print base and head commits + echo "Base commit: $BASE_COMMIT" + echo "Head commit: $HEAD_COMMIT" + + # Perform the diff to check for non-documentation changes + if git diff --name-only $BASE_COMMIT $HEAD_COMMIT -- | grep -vE '\.(md)$'; then + echo "doc_only=false" >> $GITHUB_ENV + echo "doc_only=false" >> $GITHUB_OUTPUT + else + echo "doc_only=true" >> $GITHUB_ENV + echo "doc_only=true" >> $GITHUB_OUTPUT + fi + Start-Runner: name: Start self-hosted EC2 runner runs-on: ubuntu-latest + needs: + - Check-Changes + if: ${{ needs.Check-Changes.outputs.doc_only == 'false' }} # Run only if there are non-documentation changes outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -56,7 +94,10 @@ jobs: Test-Marqo: name: Run Unit Tests - needs: Start-Runner # required to start the main job when the runner is ready + needs: + - Check-Changes # required to start the main job when the runner is ready + - Start-Runner # required to get output from the start-runner job + if: ${{ needs.Check-Changes.outputs.doc_only == 'false' }} # Run only if there are non-documentation changes runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner environment: marqo-test-suite steps: @@ -226,8 +267,9 @@ jobs: needs: - Start-Runner # required to get output from the start-runner job - Test-Marqo # required to wait when the main job is done + - Check-Changes runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + if: ${{ needs.start-runner.outputs.label }} # Only stop the runner if it was started steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1