Merge branch 'main' of https://github.com/TileDB-Inc/TileDB-Vector-Se…

…arch into jparismorgan/backwards-compat

.clang-format

@@ -0,0 +1,100 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments:
+  Kind: Always
+  OverEmptyLines: 0
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:   
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: false
+BreakConstructorInitializersBeforeComma: true
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+BreakBeforeInheritanceComma: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeCategories: 
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentRequiresClause: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+InsertNewlineAtEOF: true
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+RequiresClausePosition: OwnLine
+RequiresExpressionIndentation: OuterScope
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...

.github/workflows/check-formatting.yml

@@ -0,0 +1,27 @@
+name: Check formatting
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '_quarto.yml'
+      - 'quarto-materials/*'
+      - '**/.md'
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - '_quarto.yml'
+      - 'quarto-materials/*'
+      - '**/.md'
+
+jobs:
+  check-formatting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Check clang-format'
+        run: |
+          source $GITHUB_WORKSPACE/scripts/ci/check_clang_format.sh
+        shell: bash

.github/workflows/ci.yml

@@ -1,24 +1,24 @@
-# name: CI
+name: CI
 
-# on:
-#   push:
-#     branches: [main]
-#   pull_request:
-#     branches: [main]
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
 
-# jobs:
-#   Run-Tests:
-#     strategy:
-#       matrix:
-#         os: [ubuntu-latest]
-#     runs-on: ${{ matrix.os }}
-#     steps:
-#       - name: Install OpenBLAS
-#         run: sudo apt install libopenblas-dev
-#       - uses: actions/checkout@v3
-#       - name: Configure CMake
-#         run: cmake -S ./src -B ./src/build -DCMAKE_BUILD_TYPE=Debug -DTILEDB_VS_ENABLE_BLAS=ON
-#       - name: Build
-#         run: cmake --build ./src/build -j3
-#       - name: Run Tests
-#         run: cmake --build ./src/build --target check-ci
+jobs:
+  run-tests:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Install OpenBLAS
+        run: sudo apt install libopenblas-dev
+      - uses: actions/checkout@v3
+      - name: Configure CMake
+        run: cmake -S ./src -B ./src/build -DCMAKE_BUILD_TYPE=Debug -DTILEDB_VS_ENABLE_BLAS=ON
+      - name: Build
+        run: cmake --build ./src/build -j3
+      - name: Run Tests
+        run: cmake --build ./src/build --target check-ci

.gitignore

@@ -101,5 +101,5 @@ docs/.quarto
 documentation/reference/*
 documentation/reference
 
-# MacOS
-*.DS_Store
+# Mac
+*DS_Store

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.7.1
+    hooks:
+      - id: prettier
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: "v0.0.265"
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+
+  - repo: https://github.com/psf/black
+    rev: "23.3.0"
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: 'v17.0.6'
+    hooks:
+      - id: clang-format

README.md

@@ -40,16 +40,4 @@ We welcome contributions. Please see [`Building`](./documentation/Building.md) f
 development-build instructions. For large new
 features, please open an issue to discuss goals and approach in order
 to ensure a smooth PR integration and review process. All contributions
-must be licensed under the repository's [MIT License](./LICENSE).
-
-# Testing
-
-* Unit tests: `pytest`
-* Demo notebooks:
-  * ```
-    pip install -r test/ipynb/requirements.txt
-    pytest --nbmake test/ipynb
-    ```
-* Credentials:
-  * Some tests run on TileDB Cloud using your current environment variable `TILEDB_REST_TOKEN` -- you will need a valid API token for the tests to pass 
-  * For continuous integration, the token is configured for the `unittest` user and all tests should pass
+must be licensed under the repository's [MIT License](./LICENSE).

apis/python/pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
 
 [project.optional-dependencies]
 test = ["nbmake", "pytest"]
+formatting = ["pre-commit"]
 
 
 [project.urls]
@@ -50,6 +51,7 @@ write_to = "apis/python/src/tiledb/vector_search/version.py"
 
 [tool.ruff]
 extend-select = ["I"]
+ignore = ["F403", "F405", "E501", "E741"]
 
 [tool.ruff.isort]
 known-first-party = ["tiledb"]

apis/python/src/tiledb/vector_search/flat_index.py

@@ -8,6 +8,7 @@
 from tiledb.vector_search.storage_formats import (STORAGE_VERSION,
                                                   storage_formats,
                                                   validate_storage_version)
+from tiledb.vector_search.utils import add_to_group
 
 MAX_INT32 = np.iinfo(np.dtype("int32")).max
 MAX_UINT64 = np.iinfo(np.dtype("uint64")).max
@@ -165,7 +166,7 @@ def create(
             tile_order="col-major",
         )
         tiledb.Array.create(ids_uri, ids_schema)
-        group.add(ids_uri, name=ids_array_name)
+        add_to_group(group, ids_uri, ids_array_name)
 
         parts_array_rows_dim = tiledb.Dim(
             name="rows",
@@ -193,7 +194,7 @@ def create(
             tile_order="col-major",
         )
         tiledb.Array.create(parts_uri, parts_schema)
-        group.add(parts_uri, name=parts_array_name)
+        add_to_group(group, parts_uri, parts_array_name)
 
         external_id_dim = tiledb.Dim(
             name="external_id",
@@ -209,7 +210,7 @@ def create(
             allows_duplicates=False,
         )
         tiledb.Array.create(updates_array_uri, updates_schema)
-        group.add(updates_array_uri, name=updates_array_name)
+        add_to_group(group, updates_array_uri, updates_array_name)
 
         group.close()
         return FlatIndex(uri=uri, config=config)