Merge branch 'main' into logan/serialize_recursive_retriever

.github/workflows/publish_release.yml

@@ -31,10 +31,10 @@ jobs:
         run: pip install -e .
       - name: Cache tiktoken and nltk files
         shell: bash
-        run: python -c "from llama_index import get_tokenizer; get_tokenizer()"
+        run: python -c "from llama_index.core import get_tokenizer; get_tokenizer()"
       - name: Clean up zip files
         shell: bash
-        run: rm -rf llama_index/_static/nltk_cache/corpora/stopwords.zip llama_index/_static/nltk_cache/tokenizers/punkt.zip
+        run: rm -rf llama-index-core/llama_index/core/_static/nltk_cache/corpora/stopwords.zip llama-index-core/llama_index/core/_static/nltk_cache/tokenizers/punkt.zip
       - name: Build and publish to pypi
         uses: JRubics/[email protected]
         with:

.github/workflows/unit_test.yml

@@ -11,14 +11,21 @@ env:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-unit-tester
     strategy:
       # You can use PyPy versions in python-version.
       # For example, pypy-2.7 and pypy-3.8
       matrix:
-        python-version: ["3.10", "3.9", "3.8"]
+        python-version: ["3.8", "3.9", "3.10"]
     steps:
+      - name: clear space
+        env:
+          CI: true
+        shell: bash
+        run: rm -rf /opt/hostedtoolcache/*
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Set up python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -27,11 +34,19 @@ jobs:
         uses: snok/install-poetry@v1
         with:
           version: ${{ env.POETRY_VERSION }}
-      - name: Install deps
-        shell: bash
-        run: poetry install --with dev
+      - uses: pantsbuild/actions/init-pants@v5-scie-pants
+        with:
+          # v0 makes it easy to bust the cache if needed
+          # just increase the integer to start with a fresh cache
+          gha-cache-key: v0-py${{ matrix.python_version }}
+          named-caches-hash: v0-py${{ matrix.python_version }}
+          pants-python-version: ${{ matrix.python-version }}
+          pants-ci-config: pants.toml
+      - name: Check BUILD files
+        run: |
+          pants tailor --check :: -docs/::
       - name: Run testing
         env:
           CI: true
         shell: bash
-        run: poetry run make test
+        run: poetry run make -s test

.gitignore

@@ -1,156 +1,9 @@
-llama_index/_static
-.DS_Store
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-bin/
-build/
-develop-eggs/
+.pants.d/
 dist/
-downloads/
-eggs/
-.eggs/
-etc/
-include/
-lib/
-lib64/
-parts/
-sdist/
-share/
-var/
-wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-.ruff_cache
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-notebooks/
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
+migration_scripts/
 venv/
-ENV/
-env.bak/
-venv.bak/
-pyvenv.cfg
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# Jetbrains
-.idea
-modules/
-*.swp
-
-# VsCode
-.vscode
-
-# pipenv
-Pipfile
-Pipfile.lock
-
-# pyright
-pyrightconfig.json
-
-# persist dir for chromadb test
-/data/
+.ipynb_checkpoints
+.__pycache__
+dev_notebooks/
+llamaindex_registry.txt
+packages_to_bump_deduped.txt

.pre-commit-config.yaml

@@ -10,30 +10,30 @@ repos:
       - id: check-merge-conflict
       - id: check-symlinks
       - id: check-toml
-        exclude: llama_index/_static
+        exclude: llama-index-core/llama_index/core/_static
       - id: check-yaml
-        exclude: llama_index/_static
+        exclude: llama-index-core/llama_index/core/_static
       - id: detect-private-key
       - id: end-of-file-fixer
-        exclude: llama_index/_static
+        exclude: llama-index-core/llama_index/core/_static
       - id: mixed-line-ending
-        exclude: llama_index/_static
+        exclude: llama-index-core/llama_index/core/_static
       - id: trailing-whitespace
-        exclude: llama_index/_static
+        exclude: llama-index-core/llama_index/core/_static
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     rev: v0.1.5
 
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-        exclude: llama_index/_static
+        exclude: ".*poetry.lock|.*_static"
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 23.10.1
     hooks:
       - id: black-jupyter
         name: black-src
         alias: black
-        exclude: ^(docs/|llama_index/_static)
+        exclude: "^docs|.*poetry.lock|.*_static"
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.0.1
     hooks:
@@ -47,14 +47,17 @@ repos:
             "types-PyYAML",
             "types-protobuf==4.24.0.4",
           ]
-        exclude: ^(docs/|llama_index/_static)
+        exclude: ^(docs/|llama-index-core/llama_index/core/_static)
+        args:
+          [MYPYPATH=llama_index, --namespace-packages, --explicit-package-bases]
+        entry: bash -c 'cd $GIT_PREFIX'
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 23.10.1
     hooks:
       - id: black-jupyter
         name: black-docs-py
         alias: black
-        files: docs/
+        files: ^(docs/|examples/)
         # Using PEP 8's line length in docs prevents excess left/right scrolling
         args: [--line-length=79]
   - repo: https://github.com/adamchainz/blacken-docs
@@ -71,14 +74,24 @@ repos:
     rev: v3.0.3
     hooks:
       - id: prettier
-        exclude: llama_index/_static
+        exclude: llama-index-core/llama_index/core/_static
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:
       - id: codespell
         additional_dependencies: [tomli]
-        exclude: llama_index/_static
-        args: ["--ignore-words-list", "nin"]
+        exclude: |
+          (?x)^(
+            poetry.lock|
+            (\/.*?\.[\w:]+)/pyproject.toml|
+            llama-index-core/llama_index/core/_static
+          )$
+        args:
+          [
+            "--ignore-words-list",
+            "astroid,gallary,momento,narl,ot,rouge,nin,gere",
+          ]
+        entry: bash -c 'cd $GIT_PREFIX'
   - repo: https://github.com/srstevenson/nb-clean
     rev: 3.1.0
     hooks:
@@ -88,4 +101,4 @@ repos:
     rev: v0.23.1
     hooks:
       - id: toml-sort-fix
-        exclude: ^(poetry.lock|llama_index/_static)
+        exclude: ".*poetry.lock|.*_static"

BUILD

@@ -0,0 +1,3 @@
+poetry_requirements(
+    name="root",
+)

CHANGELOG.md

@@ -1,5 +1,31 @@
 # ChangeLog
 
+## [0.10.3] - 2024-02-13
+
+### Bug Fixes / Nits
+
+- Fixed passing in LLM to `as_chat_engine` (#10605)
+- Fixed system prompt formatting for anthropic (#10603)
+- Fixed elasticsearch vector store error on `__version__` (#10656)
+- Fixed import on openai pydantic program (#10657)
+- Added client back to opensearch vector store exports (#10660)
+- Fixed bug in SimpleDirectoryReader not using file loaders properly (#10655)
+- Added lazy LLM initialization to RankGPT (#10648)
+- Fixed bedrock embedding `from_credentials` passing ing the model name (#10640)
+- Added back recent changes to TelegramReader (#10625)
+
+## [0.10.0, 0.10.1] - 2024-02-12
+
+### Breaking Changes
+
+- Several changes are introduced. See the [full blog post](https://blog.llamaindex.ai/llamaindex-v0-10-838e735948f8) for complete deatils.
+
+## [0.9.48] - 2024-02-12
+
+### Bug Fixes / Nits
+
+- Add back deprecated API for BedrockEmbdding (#10581)
+
 ## [0.9.47] - 2024-02-11
 
 Last patch before v0.10!